├── .dockerignore ├── .gitattributes ├── .github ├── PULL_REQUEST_TEMPLATE.md ├── actions │ └── asdf │ │ └── action.yml ├── labeler.yml └── workflows │ ├── build-env-docker.yml │ ├── formatting.yml │ ├── golang.yml │ ├── haskell.yml │ ├── labeler.yml │ ├── protobuf-reprolang.yml │ ├── release.yml │ ├── rust.yml │ └── typescript.yml ├── .gitignore ├── .prettierignore ├── .prettierrc ├── .tool-versions ├── CHANGELOG.md ├── DESIGN.md ├── Development.md ├── LICENSE ├── README.md ├── bindings ├── go │ └── scip │ │ ├── assertions.go │ │ ├── assertions_noop.go │ │ ├── canonicalize.go │ │ ├── canonicalize_test.go │ │ ├── flatten.go │ │ ├── flatten_test.go │ │ ├── internal │ │ ├── compat_test.go │ │ ├── old_symbol_parser.go │ │ └── shared │ │ │ ├── sample_indexes.go │ │ │ └── shared.go │ │ ├── memtest │ │ ├── DO_NOT_ADD_NEW_TEST_FILES_HERE │ │ └── low_mem_test.go │ │ ├── parse.go │ │ ├── parse_test.go │ │ ├── position.go │ │ ├── position_test.go │ │ ├── sanitize.go │ │ ├── scip.pb.go │ │ ├── sort.go │ │ ├── sort_test.go │ │ ├── source_file.go │ │ ├── source_file_test.go │ │ ├── speedtest │ │ └── speedtest_main.go │ │ ├── symbol.go │ │ ├── symbol_formatter.go │ │ ├── symbol_formatter_test.go │ │ ├── symbol_parser.go │ │ ├── symbol_role.go │ │ ├── symbol_table.go │ │ ├── symbol_test.go │ │ ├── testdata │ │ └── index1.scip.gz │ │ ├── testutil │ │ ├── format.go │ │ ├── format_test.go │ │ └── snapshot_testing.go │ │ └── tools.go ├── haskell │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ ├── scip.cabal │ └── src │ │ └── Proto │ │ ├── Scip.hs │ │ └── Scip_Fields.hs ├── rust │ ├── Cargo.lock │ ├── Cargo.toml │ ├── LICENSE │ └── src │ │ ├── generated │ │ ├── mod.rs │ │ └── scip.rs │ │ ├── mod.rs │ │ └── symbol.rs └── typescript │ ├── package.json │ ├── scip.js │ ├── scip.ts │ └── tsconfig.json ├── buf.gen.yaml ├── buf.work.yaml ├── buf.yaml ├── cmd └── scip │ ├── convert.go │ ├── convert_test.go │ ├── lint.go │ ├── lint_test.go │ ├── main.go │ ├── main_test.go │ ├── option_from.go │ ├── print.go │ ├── print_test.go │ ├── snapshot.go │ ├── stats.go │ ├── test.go │ ├── test_test.go │ ├── tests │ ├── reprolang │ │ ├── .gitattributes │ │ ├── .gitignore │ │ ├── Cargo.lock │ │ ├── Cargo.toml │ │ ├── binding.gyp │ │ ├── bindings │ │ │ ├── go │ │ │ │ └── repro │ │ │ │ │ ├── ast.go │ │ │ │ │ ├── dependency.go │ │ │ │ │ ├── indexer.go │ │ │ │ │ ├── namer.go │ │ │ │ │ ├── parser.go │ │ │ │ │ ├── scip.go │ │ │ │ │ └── source_file.go │ │ │ ├── node │ │ │ │ ├── binding.cc │ │ │ │ └── index.js │ │ │ └── rust │ │ │ │ ├── build.rs │ │ │ │ └── lib.rs │ │ ├── generate-tree-sitter-parser.sh │ │ ├── grammar.js │ │ ├── package.json │ │ └── src │ │ │ ├── binding.go │ │ │ ├── binding_test.go │ │ │ ├── grammar.json │ │ │ ├── node-types.json │ │ │ ├── parser.c │ │ │ ├── tree_sitter │ │ │ └── parser.h │ │ │ └── workaround.go │ ├── snapshots │ │ ├── input │ │ │ ├── cyclic-reference │ │ │ │ ├── cycle1.repro │ │ │ │ └── cycle2.repro │ │ │ ├── diagnostics │ │ │ │ └── diagnostics.repro │ │ │ ├── duplicates │ │ │ │ └── duplicate.repro │ │ │ ├── forward-def │ │ │ │ └── forward_def.repro │ │ │ ├── global-cross-repo │ │ │ │ └── reference.repro │ │ │ ├── global-workspace │ │ │ │ └── hello.repro │ │ │ ├── implementation-cross-repo │ │ │ │ └── bird.repro │ │ │ ├── implementation │ │ │ │ └── animal.repro │ │ │ ├── local-document │ │ │ │ ├── local1.repro │ │ │ │ └── local2.repro │ │ │ ├── missing-symbol-information │ │ │ │ ├── globals.repro │ │ │ │ └── locals.repro │ │ │ └── relationships │ │ │ │ ├── defined_by.repro │ │ │ │ ├── mixed.repro │ │ │ │ ├── references.repro │ │ │ │ └── type_defines.repro │ │ └── output │ │ │ ├── cyclic-reference │ │ │ ├── cycle1.repro │ │ │ └── cycle2.repro │ │ │ ├── diagnostics │ │ │ └── diagnostics.repro │ │ │ ├── duplicates │ │ │ └── duplicate.repro │ │ │ ├── forward-def │ │ │ └── forward_def.repro │ │ │ ├── global-cross-repo │ │ │ └── reference.repro │ │ │ ├── global-workspace │ │ │ └── hello.repro │ │ │ ├── implementation-cross-repo │ │ │ └── bird.repro │ │ │ ├── implementation │ │ │ └── animal.repro │ │ │ ├── local-document │ │ │ ├── local1.repro │ │ │ └── local2.repro │ │ │ ├── missing-symbol-information │ │ │ ├── globals.repro │ │ │ └── locals.repro │ │ │ └── relationships │ │ │ ├── defined_by.repro │ │ │ ├── mixed.repro │ │ │ ├── references.repro │ │ │ └── type_defines.repro │ └── test_cmd │ │ ├── diagnostics │ │ ├── fails-incorrect-diagnostic.repro │ │ ├── fails-no-diagnostic.repro │ │ └── passes.repro │ │ ├── ranges │ │ ├── fails.repro │ │ └── passes.repro │ │ └── roles │ │ ├── fails-wrong-role.repro │ │ ├── fails-wrong-symbol.repro │ │ └── passes.repro │ └── version.txt ├── dev ├── Dockerfile.bindings ├── build-docker-environment.sh ├── docker-entrypoint.sh ├── generate-all-in-docker.sh ├── proto-generate.sh ├── publish-release.sh └── sample_indexes │ ├── .gitignore │ └── indexes-metadata.json ├── docs ├── CLI.md ├── scip.md ├── scip.sprig └── test_file_format.md ├── go.mod ├── go.sum ├── package.json ├── renovate.json ├── scip.proto └── yarn.lock /.dockerignore: -------------------------------------------------------------------------------- 1 | # This file should be kept in sync with .gitignore 2 | 3 | # Binaries for programs and plugins 4 | *.exe 5 | *.exe~ 6 | *.dll 7 | *.so 8 | *.dylib 9 | 10 | # Editor directories 11 | .idea/ 12 | .vscode/ 13 | 14 | # Test binary, built with `go test -c` 15 | *.test 16 | 17 | # CLI binary 18 | /scip 19 | 20 | # Output of the go coverage tool, specifically when used with LiteIDE 21 | *.out 22 | 23 | **/node_modules/ 24 | .bin/ 25 | **/target/ 26 | 27 | # Dependency directories (remove the comment below to include it) 28 | # vendor/ 29 | dist-newstyle/ 30 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | bindings/typescript/scip.ts linguist-generated=true 2 | bindings/go/scip/scip.pb.go linguist-generated=true 3 | bindings/rust/src/generated/scip.rs linguist-generated=true 4 | bindings/haskell/src/Proto/**.hs linguist-generated=true 5 | docs/scip.md linguist-generated=true 6 | yarn.lock linguist-generated=true 7 | cmd/tests/reprolang/src/grammar.json linguist-generated=true 8 | cmd/tests/reprolang/src/node-types.json linguist-generated=true 9 | cmd/tests/reprolang/src/parser.c linguist-generated=true 10 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Test plan 2 | -------------------------------------------------------------------------------- /.github/actions/asdf/action.yml: -------------------------------------------------------------------------------- 1 | name: asdf 2 | description: Installs system dependencies (Go, Rust, Nodejs) from .tool-versions file 3 | on: 4 | workflow_call: 5 | inputs: 6 | js: 7 | required: false 8 | type: boolean 9 | golang: 10 | required: false 11 | type: boolean 12 | rust: 13 | required: false 14 | type: boolean 15 | runs: 16 | using: 'composite' 17 | steps: 18 | # Avoid using separate GitHub Actions for installing toolchains for 19 | # Go, Rust etc. Maintaining all toolchain versions with asdf using 20 | # a single .tool-versions file is easier. 21 | - name: Install asdf 22 | shell: bash 23 | run: | 24 | set -e 25 | git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch v0.10.0 --depth=1 26 | echo $HOME/.asdf/bin >> $GITHUB_PATH 27 | echo $HOME/.asdf/shims >> $GITHUB_PATH 28 | - name: Install JS 29 | if: ${{ inputs.js }} 30 | shell: bash 31 | run: | 32 | asdf plugin-add yarn 33 | asdf install yarn 34 | asdf plugin-add nodejs 35 | asdf install nodejs 36 | - name: Install Rust 37 | if: ${{ inputs.rust }} 38 | shell: bash 39 | run: | 40 | RUST_WITHOUT=rust-docs asdf plugin-add rust https://github.com/asdf-community/asdf-rust.git 41 | asdf install rust 42 | - name: Install Go 43 | if: ${{ inputs.golang }} 44 | shell: bash 45 | run: | 46 | asdf plugin-add golang 47 | asdf install golang 48 | -------------------------------------------------------------------------------- /.github/labeler.yml: -------------------------------------------------------------------------------- 1 | team/graph: 2 | - '/.*/' 3 | graph/scip: 4 | - '/.*/' 5 | -------------------------------------------------------------------------------- /.github/workflows/build-env-docker.yml: -------------------------------------------------------------------------------- 1 | # If this workflow stops working, first consult the documentation page where it was copied from. 2 | # https://docs.github.com/en/actions/publishing-packages/publishing-docker-images#publishing-images-to-github-packages 3 | 4 | name: Create and publish a Docker image for bindings build environment 5 | 6 | on: 7 | push: 8 | branches: ['main'] 9 | 10 | concurrency: 11 | group: docker-build 12 | cancel-in-progress: true 13 | 14 | env: 15 | REGISTRY: ghcr.io 16 | IMAGE_NAME: ghcr.io/sourcegraph/scip-bindings-env 17 | 18 | jobs: 19 | docker_release_build: 20 | runs-on: ubuntu-latest 21 | # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. 22 | permissions: 23 | contents: read 24 | packages: write 25 | attestations: write 26 | id-token: write 27 | strategy: 28 | fail-fast: false 29 | matrix: 30 | platform: 31 | - linux/amd64 32 | - linux/arm64 33 | steps: 34 | - name: Checkout 35 | uses: actions/checkout@v3 36 | 37 | - name: Docker meta 38 | id: meta 39 | uses: docker/metadata-action@v4 40 | with: 41 | images: ${{ env.IMAGE_NAME }} 42 | tags: | 43 | type=raw,value=latest 44 | 45 | - name: Set up QEMU 46 | uses: docker/setup-qemu-action@v2 47 | 48 | - name: Set up Docker Buildx 49 | uses: docker/setup-buildx-action@v2 50 | 51 | - name: Log in to the Container registry 52 | uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 53 | with: 54 | registry: ${{ env.REGISTRY }} 55 | username: ${{ github.actor }} 56 | password: ${{ secrets.GITHUB_TOKEN }} 57 | 58 | - name: Build and push by digest 59 | id: build 60 | uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 61 | with: 62 | file: dev/Dockerfile.bindings 63 | push: true 64 | platforms: ${{ matrix.platform }} 65 | labels: ${{ steps.meta.outputs.labels }} 66 | outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true 67 | cache-from: type=registry,ref=${{ env.IMAGE_NAME}}:latest 68 | cache-to: type=inline 69 | 70 | - name: Export digest 71 | run: | 72 | mkdir -p /tmp/digests 73 | digest="${{ steps.build.outputs.digest }}" 74 | touch "/tmp/digests/${digest#sha256:}" 75 | DIGEST_SUFFIX=${{ matrix.platform }} 76 | DIGEST_SUFFIX="${DIGEST_SUFFIX//\//_}" 77 | echo "DIGEST_SUFFIX=$DIGEST_SUFFIX" >> $GITHUB_ENV 78 | 79 | - name: Upload digest 80 | uses: actions/upload-artifact@v4 81 | with: 82 | name: digests-${{ env.DIGEST_SUFFIX }} 83 | path: /tmp/digests/* 84 | if-no-files-found: error 85 | retention-days: 1 86 | 87 | - name: Generate artifact attestation 88 | uses: actions/attest-build-provenance@v1 89 | with: 90 | subject-name: ${{ env.IMAGE_NAME}} 91 | subject-digest: ${{ steps.build.outputs.digest }} 92 | push-to-registry: true 93 | 94 | docker_release_merge: 95 | runs-on: ubuntu-latest 96 | needs: [docker_release_build] 97 | # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. 98 | permissions: 99 | contents: read 100 | packages: write 101 | attestations: write 102 | id-token: write 103 | steps: 104 | - name: Log in to the Container registry 105 | uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 106 | with: 107 | registry: ${{ env.REGISTRY }} 108 | username: ${{ github.actor }} 109 | password: ${{ secrets.GITHUB_TOKEN }} 110 | 111 | - name: Download digests 112 | uses: actions/download-artifact@v4 113 | with: 114 | pattern: digests-* 115 | merge-multiple: true 116 | path: /tmp/digests 117 | 118 | - name: Set up Docker Buildx 119 | uses: docker/setup-buildx-action@v2 120 | 121 | - name: Docker meta 122 | id: meta 123 | uses: docker/metadata-action@v4 124 | with: 125 | images: ${{ env.IMAGE_NAME }} 126 | tags: | 127 | type=raw,value=latest 128 | 129 | - name: Create manifest list and push 130 | working-directory: /tmp/digests 131 | run: | 132 | docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ 133 | $(printf '${{ env.IMAGE_NAME }}@sha256:%s ' *) 134 | 135 | - name: Inspect image 136 | run: | 137 | docker buildx imagetools inspect ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} 138 | -------------------------------------------------------------------------------- /.github/workflows/formatting.yml: -------------------------------------------------------------------------------- 1 | name: Formatting 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - '.github/workflows/**' 7 | - '**.md' 8 | - '**.yml' 9 | - '**.yaml' 10 | - '**.js' 11 | - '**.json' 12 | - '**.ts' 13 | 14 | jobs: 15 | prettier: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v3 19 | - run: yarn install 20 | - run: yarn run prettier-check 21 | -------------------------------------------------------------------------------- /.github/workflows/golang.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - '.github/workflows/**' 7 | - '**.go' 8 | - '**/go.mod' 9 | - '**/go.sum' 10 | - 'cmd/scip/**' 11 | - 'docs/CLI.md' 12 | 13 | jobs: 14 | go-test: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v3 18 | - uses: ./.github/actions/asdf 19 | with: 20 | golang: true 21 | - run: go test ./... -v -tags asserts 22 | -------------------------------------------------------------------------------- /.github/workflows/haskell.yml: -------------------------------------------------------------------------------- 1 | name: Haskell 2 | on: 3 | pull_request: 4 | paths: 5 | - '.github/workflows/**' 6 | - '**.hs' 7 | - '**.cabal' 8 | 9 | jobs: 10 | haskell-typecheck: 11 | strategy: 12 | fail-fast: false 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout code 16 | uses: actions/checkout@v3 17 | - name: Set up Haskell 18 | uses: haskell/actions/setup@v2 19 | with: 20 | ghc-version: '8.10' 21 | cabal-version: 'latest' 22 | - name: Build source 23 | run: cabal build --ghc-options='-j2 +RTS -A32m' 24 | working-directory: bindings/haskell 25 | -------------------------------------------------------------------------------- /.github/workflows/labeler.yml: -------------------------------------------------------------------------------- 1 | name: 'Issue Labeler' 2 | on: 3 | issues: 4 | types: [opened, edited] 5 | 6 | permissions: 7 | issues: write 8 | contents: read 9 | 10 | jobs: 11 | triage: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: github/issue-labeler@v3.4 15 | with: 16 | configuration-path: .github/labeler.yml 17 | enable-versioned-regex: 0 18 | repo-token: ${{ github.token }} 19 | -------------------------------------------------------------------------------- /.github/workflows/protobuf-reprolang.yml: -------------------------------------------------------------------------------- 1 | name: Generated code is up to date 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - '.github/workflows/**' 7 | - 'docs/**' 8 | - 'bindings/**' 9 | - 'scip.proto' 10 | - 'buf**' 11 | - '.tool-versions' 12 | - 'dev/proto-generate.sh' 13 | - 'dev/proto-generate-in-docker.sh' 14 | - 'Dockerfile.bindings' 15 | - 'cmd/scip/tests/reprolang/**' 16 | 17 | jobs: 18 | gen-up-to-date: 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v3 22 | - name: Log in to the Container registry 23 | uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 24 | with: 25 | registry: ghcr.io 26 | username: ${{ github.actor }} 27 | password: ${{ secrets.GITHUB_TOKEN }} 28 | 29 | - run: docker pull ghcr.io/sourcegraph/scip-bindings-env:latest || echo "no suitable cache" 30 | 31 | - name: Regenerate protobuf bindings and reprolang parser 32 | run: | 33 | # We're changing the owner of the checkout folder to a particular user id, 34 | # matching the user id of `asdf` user we create inside the docker container. 35 | sudo chown -R 1001:1001 . && ./dev/generate-all-in-docker.sh 36 | 37 | - run: git diff --exit-code 38 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | release-crate: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v3 12 | - uses: ./.github/actions/asdf 13 | with: 14 | rust: true 15 | - run: cargo publish --token '${{ secrets.CRATES_TOKEN }}' 16 | working-directory: bindings/rust 17 | 18 | build-go-binaries: 19 | runs-on: ubuntu-latest 20 | 21 | strategy: 22 | matrix: 23 | goos: [linux, darwin] 24 | goarch: [amd64, arm64] 25 | 26 | steps: 27 | - uses: actions/checkout@v3 28 | - uses: wangyoucao577/go-release-action@v1.40 29 | with: 30 | github_token: ${{ secrets.GITHUB_TOKEN }} 31 | goos: ${{ matrix.goos }} 32 | goarch: ${{ matrix.goarch }} 33 | sha256sum: true 34 | project_path: cmd/scip 35 | binary_name: scip 36 | ldflags: "-X 'main.Reproducible=true'" 37 | asset_name: scip-${{ matrix.goos }}-${{ matrix.goarch }} 38 | extra_files: LICENSE 39 | -------------------------------------------------------------------------------- /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - '.github/workflows/**' 7 | - '**.rs' 8 | - '**/Cargo.toml' 9 | - '**/Cargo.lock' 10 | 11 | jobs: 12 | rust-typecheck: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v3 16 | - uses: ./.github/actions/asdf 17 | with: 18 | rust: true 19 | - run: cargo test 20 | working-directory: bindings/rust 21 | - run: cargo check 22 | working-directory: cmd/scip/tests/reprolang 23 | -------------------------------------------------------------------------------- /.github/workflows/typescript.yml: -------------------------------------------------------------------------------- 1 | name: TypeScript 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - '.github/workflows/**' 7 | - '**.ts' 8 | - '**/package.json' 9 | - '**/tsconfig.json' 10 | - '**/yarn.lock' 11 | 12 | jobs: 13 | typescript-typecheck: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | - uses: ./.github/actions/asdf 18 | with: 19 | js: true 20 | - run: yarn install 21 | - name: Typecheck TypeScript 22 | run: yarn run build 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Editor directories 9 | .idea/ 10 | .vscode/ 11 | 12 | # Test binary, built with `go test -c` 13 | *.test 14 | 15 | # CLI binary 16 | /scip 17 | 18 | # Output of the go coverage tool, specifically when used with LiteIDE 19 | *.out 20 | 21 | **/node_modules/ 22 | .bin/ 23 | **/target/ 24 | 25 | # Dependency directories (remove the comment below to include it) 26 | # vendor/ 27 | dist-newstyle/ 28 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | reprolang/target 2 | reprolang/src/grammar.json 3 | reprolang/src/node-types.json 4 | bindings/typescript 5 | docs/scip.md 6 | .bin 7 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "trailingComma": "es5", 4 | "singleQuote": true, 5 | "useTabs": false, 6 | "arrowParens": "avoid" 7 | } 8 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | golang 1.22.0 2 | nodejs 16.20.2 3 | shellcheck 0.7.1 4 | yarn 1.22.22 5 | rust 1.81.0 6 | python 3.11.9 7 | -------------------------------------------------------------------------------- /DESIGN.md: -------------------------------------------------------------------------------- 1 | # Design rationale for SCIP 2 | 3 | Sourcegraph supports viewing and navigating code for 4 | many different programming languages. 5 | This is similar to an IDE, except for the actual "editor" part. 6 | 7 | The design of SCIP is based on this primary motivating use case, 8 | as well as for fixing the pain points we encountered with using LSIF.[^1] 9 | 10 | SCIP is meant to be a _transmission_ format for sending 11 | data from some producers to some consumers 12 | -- it is not meant as a _storage_ format for querying. 13 | Ideally, producers should be able to directly output SCIP 14 | instead of going through an intermediate format 15 | and then converting to SCIP for transmission. 16 | 17 | [^1]: 18 | Sourcegraph historically supported LSIF uploads 19 | as well as maintained our own LSIF indexers, 20 | but we ran into issues of development velocity, 21 | debugging, as well as indexer performance bottlenecks. 22 | 23 | ## Goals 24 | 25 | ### Primary goals 26 | 27 | - Support code navigation at the fidelity of state-of-the-art IDEs. 28 | - Why: We want people to have an excellent experience navigating 29 | their code inside Sourcegraph itself. 30 | - Ease of writing indexers (i.e. producers). 31 | - Adding cross-repo navigation support should be easy. 32 | - Why: Scaling for Sourcegraph customers using code spread across many repos, 33 | as well as supporting code navigation for package ecosystems. 34 | - Adding file-level incrementality should be easy. 35 | - Why: Scaling for large monorepos. 36 | - Making the indexer parallel should be easy 37 | - Why: Scaling for large monorepos. 38 | - Writing indexers in different languages should be feasible 39 | - Why: Generally, tooling for a language tends to be implemented 40 | in the same language or an adjacent one; it would be impractical 41 | to expect indexers to settle on a common implementation language. 42 | - Robustness against indexer bugs: Incorrect code nav data for a certain entity 43 | should have a limited blast radius. 44 | - Ease of debugging. 45 | 46 | ## Non-goals 47 | 48 | - Support use cases involving code modifications. 49 | - Why: Sourcegraph's code search and navigation has historically 50 | focused on read-only use cases. Adding support for code modifications 51 | introduces more complexity. 52 | While the same data can be used for tools for refactoring tools 53 | and IDE tooling, supporting those is not the focus of SCIP. 54 | - Ease of writing consumers. 55 | - Why: We expect the number of SCIP producers to be much higher 56 | than the number of consumers, 57 | so it makes sense to optimize for producers. 58 | - Be as compact as possible in uncompressed form.[^2] 59 | - Why: Modern general-purpose compression formats like gzip 60 | and zstd are already very good in terms of both compression 61 | speed and compression ratio.[^3] 62 | - Support efficient code navigation by itself. 63 | 64 | - Why: Code navigation fundamentally requires some form of bidirectional 65 | lookup which is best served by a query engine. 66 | 67 | For example, finding subclasses and superclasses are dual operations; 68 | supporting both across different indexes (for cross-repository navigation) 69 | requires some way of connecting the data together anyways. 70 | However, if the consumer is capable of supporting that, 71 | then recording bidirectional links in index data is not useful. 72 | 73 | [^2]: The only compromise on this decision is in the choice of representation of source ranges, where benchmarks showed that using a variable length integer array encoding provided significant savings compared to a message-based encoding, even after compression. 74 | 75 | [^3]: In practice, SCIP data tends to be have a compression ratio around in the range of 10%-20%, as modern compressors are very good at de-duplicating away the repetitive textual symbols. 76 | 77 | ## Core design decisions 78 | 79 | ### Using Protobuf for the schema 80 | 81 | - Relatively compact binary format, reducing I/O overhead. 82 | - Protobuf supports easy code generation. 83 | - Many languages have Protobuf code generators. 84 | - TLV format enables streaming reads and writes 85 | as well as merging by concatenation. 86 | - Rules for maintaining forward and backward compatibility 87 | are easily understood. 88 | 89 | ### Using strings for IDs 90 | 91 | Hash tables are a core data type used in compilers and 92 | hence are likely to be useful in indexers generally. 93 | 94 | String types in mainstream languages support equality and hashing, 95 | where other objects may not be. 96 | 97 | ### Avoid direct encoding of graphs 98 | 99 | One very tempting design for code navigation data is to 100 | think of all semantic entities as nodes and relationships between 101 | entities as edges, and to simply record ALL data 102 | using an adjacency list based graph representation. 103 | 104 | This is conceptually appealing, 105 | but is not desirable for a few reasons: 106 | 107 | - It encourages a wholesale approach to writing 108 | indexers as it involves merging all the data together. 109 | Such indexers are less likely to be able 110 | to easily support parallelism. 111 | - This potentially requires keeping a lot of data of memory 112 | at indexing time. Ideally, an indexer should be able 113 | to load parts of the codebase, 114 | append index data for that part to an open file, 115 | and then move on to the next part having cleared all memory 116 | being used from the previous iteration. 117 | - It potentially requires keeping a lot of data in memory 118 | on the consumer side. 119 | 120 | Instead, the approach to using documents and arrays 121 | helps colocate relevant data and naturally allows for streaming 122 | if the underlying data format allows streaming. 123 | 124 | ### Avoiding integer IDs 125 | 126 | This includes avoiding structures like a symbol table 127 | mapping string IDs to integers and mostly using the integer 128 | IDs in raw data. 129 | 130 | As far as we know, the integer IDs in LSIF are primarily present as 131 | an ad-hoc compression scheme due to the verbosity of JSON 132 | and LSIF's graph-based encoding scheme. 133 | 134 | Avoiding integer IDs helps with limiting the blast radius 135 | of indexer bugs. With LSIF, we've had off-by-one bugs in indexers 136 | cause code navigation to fail repo-wide. 137 | 138 | This also helps with debugging, as the raw data itself 139 | can be inspected much more easily without needing 140 | a lot of surrounding context. 141 | -------------------------------------------------------------------------------- /Development.md: -------------------------------------------------------------------------------- 1 | # Developing SCIP 2 | 3 | - [Project structure](#project-structure) 4 | - [Code generation](#code-generation) 5 | - [Debugging](#debugging) 6 | - [Benchmarking](#benchmarking) 7 | - [Testing and adding new SCIP semantics](#testing-and-adding-new-scip-semantics) 8 | - [Release a new version](#release-a-new-version) 9 | 10 | ## Project structure 11 | 12 | - [bindings/](./bindings/): Contains a mix of generated and hand-written 13 | bindings for different languages. 14 | - The TypeScript, Rust and Haskell bindings are auto-generated. 15 | - The Go bindings include protoc-generated code as well as extra 16 | functionality, such as for converting a SCIP index into an LSIF index. 17 | This is used by the CLI below as well as the 18 | [Sourcegraph CLI](https://github.com/sourcegraph/src-cli). 19 | - [cmd/scip](./cmd/scip): CLI for SCIP. 20 | - [cmd/scip/tests/](./cmd/scip/tests/): Test data and packages for SCIP. 21 | - [cmd/scip/tests/reprolang/](./cmd/scip/tests/reprolang/): A verbose, small language 22 | which consists of declarations, references, imports and other minor bits 23 | of functionality, which is used to test the SCIP CLI. The language is 24 | defined using a [tree-sitter grammar](cmd/scip/tests/reprolang/grammar.js). 25 | This functionality not meant for use outside of this repository. 26 | - [docs/](./docs/): Auto-generated documentation. 27 | 28 | ## Code generation 29 | 30 | 1. Regenerating definitions after changing the schema in [scip.proto](./scip.proto). 31 | 32 | `./dev/generate-all-in-docker.sh` 33 | 34 | We provide a script that sets up the correct build environment in Docker 35 | and runs the necessary regeneration steps. 36 | 37 | Both the proto bindings and reprolang parser are generated. 38 | The only dependency you need is Docker. 39 | 40 | 2. Regenerating snapshots after making changes to the CLI. 41 | ``` 42 | go test ./cmd/scip -update-snapshots 43 | ``` 44 | 3. Regenerating parser for Repro after editing its grammar. 45 | ``` 46 | cd cmd/scip/tests/reprolang 47 | ./generate-tree-sitter-parser.sh 48 | ``` 49 | 50 | ## Debugging 51 | 52 | Protobuf output can be inspected using `scip print`: 53 | 54 | ``` 55 | scip print /path/to/index.scip 56 | ``` 57 | 58 | This may be a bit verbose. The default Protobuf output is more compact, 59 | and can be inspected using `protoc`: 60 | 61 | ``` 62 | protoc --decode=scip.Index -I /path/to/scip scip.proto < index.scip 63 | ``` 64 | 65 | There is also a `lint` subcommand which performs various well-formedness 66 | checks on a SCIP index. It is meant primarily for people working on a SCIP indexer, 67 | and is not recommended for use in other settings. 68 | 69 | ``` 70 | scip lint /path/to/index.scip 71 | ``` 72 | 73 | ## Benchmarking 74 | 75 | For benchmarks, one can put test SCIP indexes under `dev/sample_indexes`. 76 | 77 | Sourcegraph teammates can download several large indexes 78 | from this [Google drive folder](https://drive.google.com/drive/folders/1z62Se7eHaa5T89a16-y7s0Z1qbRY4VCg). 79 | 80 | After that you can run: 81 | 82 | ```bash 83 | go run ./bindings/go/scip/speedtest 84 | ``` 85 | 86 | to see the results. 87 | 88 | Make sure to share benchmark results when making changes to 89 | the symbol parsing logic. 90 | 91 | ## Testing and adding new SCIP semantics 92 | 93 | It is helpful to use reprolang to check the existing code navigation behavior, 94 | to design new code navigation behavior, 95 | or to investigate the effect of the SCIP to LSIF desugaring. 96 | The LSIF index for reprolang code is much smaller, 97 | which aids debugging. 98 | 99 | To do this, add a test file (and implement any new functionality) first. 100 | Then, regenerate the LSIF index with absolute paths. 101 | 102 | ```bash 103 | go test ./cmd/scip -update-snapshots -debug-snapshot-abspaths 104 | ``` 105 | 106 | The LSIF index can be uploaded to a local Sourcegraph instance using: 107 | 108 | ```bash 109 | PACKAGE=MY_PACKAGE_NAME SRC_ACCESS_TOKEN=MY_TOKEN SRC_ENDPOINT=https://sourcegraph.test:3443 src code-intel upload -file="cmd/scip/tests/snapshots/output/$PACKAGE/dump.lsif" -root="cmd/scip/tests/snapshots/input/$PACKAGE" 110 | ``` 111 | 112 | ## Release a new version 113 | 114 | First, add release notes to the [CHANGELOG](CHANGELOG.md). Next, update the 115 | version in `cmd/scip/version.txt`, `bindings/rust/Cargo.toml`, 116 | `bindings/rust/Cargo.lock`, and `docs/CLI.md` 117 | 118 | After landing a commit with those two changes, run the release script: 119 | (requires the [GitHub CLI](https://cli.github.com/)) 120 | 121 | ```bash 122 | NEW_VERSION="M.N.P" ./dev/publish-release.sh 123 | ``` 124 | 125 | Once the release is created, the artifacts will be built and uploaded 126 | automatically by the [release action](/.github/workflows/release.yml). 127 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SCIP Code Intelligence Protocol 2 | 3 | SCIP (pronunciation: "skip") is a language-agnostic protocol 4 | for indexing source code, 5 | which can be used to power code navigation functionality 6 | such as Go to definition, Find references, and Find implementations. 7 | 8 | This repository includes: 9 | 10 | - A [Protobuf schema for SCIP](./scip.proto). 11 | - Rich Go and Rust bindings for SCIP: These include many utility functions 12 | to help build tooling on top of SCIP. 13 | - Auto-generated bindings for TypeScript and Haskell. 14 | - The [`scip` CLI](./docs/CLI.md), which makes SCIP indexes 15 | a breeze to work with. 16 | 17 | If you're interested in better understanding the motivation behind SCIP, 18 | check out the [announcement blog post](https://about.sourcegraph.com/blog/announcing-scip) and the [design doc](./DESIGN.md). 19 | 20 | If you're interested in writing a new indexer that emits SCIP, 21 | check out our documentation on 22 | [how to write an indexer](https://docs.sourcegraph.com/code_intelligence/explanations/writing_an_indexer). 23 | Also, check out the [Debugging section][] in the Development docs. 24 | 25 | If you're interested in consuming SCIP data, 26 | you can either use one of the [provided language bindings](https://github.com/sourcegraph/scip/tree/main/bindings), 27 | or generate code for the [SCIP Protobuf schema](./scip.proto) 28 | using the Protobuf toolchain for your language ecosystem. 29 | Also, check out the [Debugging section][] in the Development docs. 30 | 31 | [debugging section]: ./Development.md#debugging 32 | 33 | ## Tools using SCIP 34 | 35 | Several indexers currently emit SCIP data: 36 | 37 | - [scip-java](https://github.com/sourcegraph/scip-java): Java, Scala, Kotlin 38 | - [scip-typescript](https://github.com/sourcegraph/scip-typescript): TypeScript, JavaScript 39 | - [rust-analyzer](https://github.com/rust-lang/rust-analyzer): Rust 40 | - [scip-clang](https://github.com/sourcegraph/scip-clang): C++, C 41 | - [scip-ruby](https://github.com/sourcegraph/scip-ruby): Ruby 42 | - [scip-python](https://github.com/sourcegraph/scip-python): Python 43 | - [scip-dotnet](https://github.com/sourcegraph/scip-dotnet): C#, Visual Basic 44 | - [scip-dart](https://github.com/Workiva/scip-dart): Dart 45 | - [scip-php](https://github.com/davidrjenni/scip-php): PHP 46 | 47 | For more details about indexers, including LSIF-based indexers, 48 | see the [Sourcegraph documentation](https://docs.sourcegraph.com/code_navigation/references/indexers). 49 | 50 | Other tools which use SCIP include the [Sourcegraph CLI](https://github.com/sourcegraph/src-cli), 51 | and the SCIP CLI in this repo. 52 | 53 | ## Installing the `scip` CLI 54 | 55 | You can find binaries for the `scip` CLI tool [here](https://github.com/sourcegraph/scip/releases). 56 | You can also compile a binary locally using: 57 | 58 | ```sh 59 | git clone https://github.com/sourcegraph/scip.git --depth=1 60 | cd scip 61 | go build ./cmd/scip 62 | ``` 63 | 64 | You can consult the [CLI reference](docs/CLI.md) or `--help` for usage information. 65 | 66 | ## Contributing 67 | 68 | We welcome questions, suggestions as well as code and docs contributions. 69 | 70 | For submitting contributions, check out [Development.md](./Development.md) 71 | to better understand project structure and common workflows. 72 | 73 | Contributors should abide by the [Sourcegraph Code of Conduct](https://handbook.sourcegraph.com/company-info-and-process/communication/code_of_conduct/). 74 | -------------------------------------------------------------------------------- /bindings/go/scip/assertions.go: -------------------------------------------------------------------------------- 1 | //go:build asserts 2 | 3 | package scip 4 | 5 | func assert(cond bool, msg string) { 6 | if !cond { 7 | panic(msg) 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /bindings/go/scip/assertions_noop.go: -------------------------------------------------------------------------------- 1 | //go:build !asserts 2 | 3 | package scip 4 | 5 | // assert is a noop in release builds - the implementation is in assertions.go 6 | func assert(cond bool, msg string) {} 7 | -------------------------------------------------------------------------------- /bindings/go/scip/canonicalize.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | // CanonicalizeDocument deterministically sorts and merges fields of the given document. 4 | // 5 | // Post-conditions: 6 | // 1. The Occurrences field only contains those with well-formed ranges 7 | // (length 3 or 4, potentially empty). 8 | // 2. The Occurrences field is sorted in ascending order of ranges based on 9 | // Range.CompareStrict 10 | // 3. The Symbols field is sorted in ascending order based on the symbol name, 11 | // and SymbolInformation values for the same name will have been merged. 12 | func CanonicalizeDocument(document *Document) *Document { 13 | document.Occurrences = CanonicalizeOccurrences(document.Occurrences) 14 | document.Symbols = CanonicalizeSymbols(document.Symbols) 15 | return SanitizeDocument(document) 16 | } 17 | 18 | // CanonicalizeOccurrences deterministically re-orders the fields of the given occurrence slice. 19 | func CanonicalizeOccurrences(occurrences []*Occurrence) []*Occurrence { 20 | canonicalized := make([]*Occurrence, 0, len(occurrences)) 21 | for _, occurrence := range FlattenOccurrences(RemoveIllegalOccurrences(occurrences)) { 22 | canonicalized = append(canonicalized, CanonicalizeOccurrence(occurrence)) 23 | } 24 | 25 | return SortOccurrences(canonicalized) 26 | } 27 | 28 | // RemoveIllegalOccurrences removes all occurrences that do not include a range. This is 29 | // emitted by some indexers and will silently crash a downstream process, including further 30 | // canonicalization, when trying to convert an empty slice into a valid range. 31 | func RemoveIllegalOccurrences(occurrences []*Occurrence) []*Occurrence { 32 | filtered := occurrences[:0] 33 | for _, occurrence := range occurrences { 34 | if len(occurrence.Range) != 3 && len(occurrence.Range) != 4 { 35 | continue 36 | } 37 | 38 | filtered = append(filtered, occurrence) 39 | } 40 | 41 | return filtered 42 | } 43 | 44 | // CanonicalizeOccurrence deterministically re-orders the fields of the given occurrence. 45 | func CanonicalizeOccurrence(occurrence *Occurrence) *Occurrence { 46 | // Express ranges as three-components if possible 47 | occurrence.Range = NewRangeUnchecked(occurrence.Range).SCIPRange() 48 | occurrence.Diagnostics = CanonicalizeDiagnostics(occurrence.Diagnostics) 49 | return occurrence 50 | } 51 | 52 | // CanonicalizeDiagnostics deterministically re-orders the fields of the given diagnostic slice. 53 | func CanonicalizeDiagnostics(diagnostics []*Diagnostic) []*Diagnostic { 54 | canonicalized := make([]*Diagnostic, 0, len(diagnostics)) 55 | for _, diagnostic := range diagnostics { 56 | canonicalized = append(canonicalized, CanonicalizeDiagnostic(diagnostic)) 57 | } 58 | 59 | return SortDiagnostics(canonicalized) 60 | } 61 | 62 | // CanonicalizeDiagnostic deterministically re-orders the fields of the given diagnostic. 63 | func CanonicalizeDiagnostic(diagnostic *Diagnostic) *Diagnostic { 64 | diagnostic.Tags = SortDiagnosticTags(diagnostic.Tags) 65 | return diagnostic 66 | } 67 | 68 | // CanonicalizeSymbols deterministically re-orders the fields of the given symbols slice. 69 | func CanonicalizeSymbols(symbols []*SymbolInformation) []*SymbolInformation { 70 | canonicalized := make([]*SymbolInformation, 0, len(symbols)) 71 | for _, symbol := range FlattenSymbols(symbols) { 72 | canonicalized = append(canonicalized, CanonicalizeSymbol(symbol)) 73 | } 74 | 75 | return SortSymbols(canonicalized) 76 | } 77 | 78 | // CanonicalizeSymbol deterministically re-orders the fields of the given symbol. 79 | func CanonicalizeSymbol(symbol *SymbolInformation) *SymbolInformation { 80 | symbol.Relationships = CanonicalizeRelationships(symbol.Relationships) 81 | return symbol 82 | } 83 | 84 | // CanonicalizeRelationships deterministically re-orders the fields of the given relationship slice. 85 | func CanonicalizeRelationships(relationships []*Relationship) []*Relationship { 86 | return SortRelationships(FlattenRelationship(relationships)) 87 | } 88 | -------------------------------------------------------------------------------- /bindings/go/scip/canonicalize_test.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | ) 8 | 9 | func TestCanonicalizeDocument(t *testing.T) { 10 | document := &Document{ 11 | RelativePath: "foo.go", 12 | Symbols: []*SymbolInformation{ 13 | {Symbol: "foo"}, 14 | {Symbol: "bar"}, 15 | {Symbol: "bonk"}, 16 | 17 | // duplicates 18 | {Symbol: "baz", Relationships: []*Relationship{{Symbol: "bazzer", IsReference: true}}}, 19 | {Symbol: "baz", Relationships: []*Relationship{{Symbol: "bazImpl", IsImplementation: true}}}, 20 | 21 | // duplicates 22 | {Symbol: "quux", Documentation: []string{"docs1"}}, 23 | {Symbol: "quux", Documentation: []string{"docs2"}}, 24 | }, 25 | Occurrences: []*Occurrence{ 26 | // illegal range (removed) 27 | {Range: []int32{}, Symbol: "foo"}, 28 | 29 | {Range: []int32{2, 1, 2, 4}, Symbol: "bar", SymbolRoles: int32(SymbolRole_Definition)}, 30 | {Range: []int32{1, 1, 1, 4}, Symbol: "foo"}, 31 | {Range: []int32{4, 1, 4, 4}, Symbol: "bonk"}, 32 | {Range: []int32{6, 1, 6, 4}, Symbol: "honk"}, 33 | 34 | // duplicates (same symbol name) 35 | {Range: []int32{5, 1, 5, 4}, Symbol: "quux", OverrideDocumentation: []string{"primo"}}, 36 | {Range: []int32{5, 1, 5, 4}, Symbol: "quux", OverrideDocumentation: []string{"secondo"}}, 37 | 38 | // duplicate (different symbol name) 39 | {Range: []int32{3, 1, 4, 4}, Symbol: "baz"}, 40 | {Range: []int32{3, 1, 4, 4}, Symbol: "bazImpl"}, 41 | }, 42 | } 43 | canonicalizedDocument := CanonicalizeDocument(document) 44 | 45 | expectedDocument := &Document{ 46 | RelativePath: "foo.go", 47 | Symbols: []*SymbolInformation{ 48 | {Symbol: "bar"}, 49 | {Symbol: "baz", Relationships: []*Relationship{{Symbol: "bazImpl", IsImplementation: true}, {Symbol: "bazzer", IsReference: true}}}, 50 | {Symbol: "bonk"}, 51 | {Symbol: "foo"}, 52 | {Symbol: "quux", Documentation: []string{"docs1", "docs2"}}, 53 | }, 54 | Occurrences: []*Occurrence{ 55 | {Range: []int32{1, 1, 4}, Symbol: "foo"}, 56 | {Range: []int32{2, 1, 4}, Symbol: "bar", SymbolRoles: int32(SymbolRole_Definition)}, 57 | {Range: []int32{3, 1, 4, 4}, Symbol: "baz"}, 58 | {Range: []int32{3, 1, 4, 4}, Symbol: "bazImpl"}, 59 | {Range: []int32{4, 1, 4}, Symbol: "bonk"}, 60 | {Range: []int32{5, 1, 4}, Symbol: "quux", OverrideDocumentation: []string{"primo", "secondo"}}, 61 | {Range: []int32{6, 1, 4}, Symbol: "honk"}, 62 | }, 63 | } 64 | if diff := cmp.Diff(expectedDocument, canonicalizedDocument, compareDocuments); diff != "" { 65 | t.Fatalf("unexpected document (-want +got):\n%s", diff) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /bindings/go/scip/flatten.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | // FlattenDocuments merges elements of the given slice with the same relative path. This allows us to make 4 | // the assumption post-canonicalization that each index has one representation of a given document path in 5 | // the database. This function returns a new slice. 6 | func FlattenDocuments(documents []*Document) []*Document { 7 | documentMap := make(map[string]*Document, len(documents)) 8 | for _, document := range documents { 9 | existing, ok := documentMap[document.RelativePath] 10 | if !ok { 11 | documentMap[document.RelativePath] = document 12 | continue 13 | } 14 | if existing.Language != document.Language { 15 | _ = 0 // TODO - warn? 16 | } 17 | 18 | existing.Symbols = append(existing.Symbols, document.Symbols...) 19 | existing.Occurrences = append(existing.Occurrences, document.Occurrences...) 20 | } 21 | 22 | flattened := make([]*Document, 0, len(documentMap)) 23 | for _, document := range documentMap { 24 | flattened = append(flattened, document) 25 | } 26 | 27 | return flattened 28 | } 29 | 30 | // FlattenSymbol merges elements of the given slice with the same symbol name. This allows us to make the 31 | // assumption post-canonicalization that each index and document refer to one symbol metadata object uniquely. 32 | // This function returns a new slice. 33 | func FlattenSymbols(symbols []*SymbolInformation) []*SymbolInformation { 34 | symbolMap := make(map[string]*SymbolInformation, len(symbols)) 35 | for _, symbol := range symbols { 36 | existing, ok := symbolMap[symbol.Symbol] 37 | if !ok { 38 | symbolMap[symbol.Symbol] = symbol 39 | continue 40 | } 41 | 42 | existing.Documentation = combineDocumentation(existing.Documentation, symbol.Documentation) 43 | existing.Relationships = append(existing.Relationships, symbol.Relationships...) 44 | } 45 | 46 | flattened := make([]*SymbolInformation, 0, len(symbolMap)) 47 | for _, symbol := range symbolMap { 48 | flattened = append(flattened, symbol) 49 | } 50 | 51 | return flattened 52 | } 53 | 54 | // FlattenOccurrences merges elements of the given slice with equivalent bounds. This function returns a new slice. 55 | func FlattenOccurrences(occurrences []*Occurrence) []*Occurrence { 56 | if len(occurrences) == 0 { 57 | return occurrences 58 | } 59 | 60 | _ = SortOccurrences(occurrences) 61 | flattened := make([]*Occurrence, 0, len(occurrences)) 62 | flattened = append(flattened, occurrences[0]) 63 | 64 | for _, occurrence := range occurrences[1:] { 65 | top := flattened[len(flattened)-1] 66 | 67 | if !rawRangesEqual(top.Range, occurrence.Range) { 68 | flattened = append(flattened, occurrence) 69 | continue 70 | } 71 | if top.Symbol != occurrence.Symbol { 72 | flattened = append(flattened, occurrence) 73 | continue 74 | } 75 | 76 | if top.SyntaxKind == SyntaxKind_UnspecifiedSyntaxKind { 77 | // Take first valid syntax kind 78 | top.SyntaxKind = occurrence.SyntaxKind 79 | } 80 | 81 | // Combine all other fields 82 | top.SymbolRoles |= occurrence.SymbolRoles 83 | top.OverrideDocumentation = append(top.OverrideDocumentation, occurrence.OverrideDocumentation...) 84 | top.Diagnostics = append(top.Diagnostics, occurrence.Diagnostics...) 85 | } 86 | 87 | return flattened 88 | } 89 | 90 | // FlattenRelationship merges elements of the given slice with equivalent symbol names. This function returns a new 91 | // slice. 92 | func FlattenRelationship(relationships []*Relationship) []*Relationship { 93 | relationshipMap := make(map[string][]*Relationship, len(relationships)) 94 | for _, relationship := range relationships { 95 | relationshipMap[relationship.Symbol] = append(relationshipMap[relationship.Symbol], relationship) 96 | } 97 | 98 | flattened := make([]*Relationship, 0, len(relationshipMap)) 99 | for _, relationships := range relationshipMap { 100 | combined := relationships[0] 101 | for _, relationship := range relationships[1:] { 102 | combined.IsReference = combined.IsReference || relationship.IsReference 103 | combined.IsImplementation = combined.IsImplementation || relationship.IsImplementation 104 | combined.IsTypeDefinition = combined.IsTypeDefinition || relationship.IsTypeDefinition 105 | combined.IsDefinition = combined.IsDefinition || relationship.IsDefinition 106 | } 107 | 108 | flattened = append(flattened, combined) 109 | } 110 | 111 | return flattened 112 | } 113 | 114 | // combineDocumentation merges documentation components from two separate symbol information objects. 115 | func combineDocumentation(existing, additional []string) []string { 116 | filtered := make([]string, 0, len(additional)) 117 | for _, s := range additional { 118 | if !stringSliceContains(existing, s) { 119 | filtered = append(filtered, s) 120 | } 121 | } 122 | 123 | return append(existing, filtered...) 124 | } 125 | 126 | func stringSliceContains(slice []string, target string) bool { 127 | for _, candidate := range slice { 128 | if target == candidate { 129 | return true 130 | } 131 | } 132 | 133 | return false 134 | } 135 | -------------------------------------------------------------------------------- /bindings/go/scip/flatten_test.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | ) 8 | 9 | func TestFlattenDocuments(t *testing.T) { 10 | s1 := []*SymbolInformation{{Symbol: "foo"}, {Symbol: "bar"}} 11 | o1 := []*Occurrence{{Range: []int32{1, 2, 3}, Symbol: "foo"}, {Range: []int32{2, 3, 4}, Symbol: "bar"}} 12 | s2 := []*SymbolInformation{{Symbol: "baz"}, {Symbol: "bonk"}} 13 | o2 := []*Occurrence{{Range: []int32{3, 4, 5}, Symbol: "baz"}, {Range: []int32{4, 5, 6}, Symbol: "bonk"}} 14 | s3 := []*SymbolInformation{{Symbol: "quux"}} 15 | o3 := []*Occurrence{{Range: []int32{5, 6, 7}, Symbol: "quux"}} 16 | 17 | documents := []*Document{ 18 | {RelativePath: "foo.go", Symbols: s1, Occurrences: o1}, 19 | {RelativePath: "bar.go", Symbols: s2, Occurrences: o2}, 20 | {RelativePath: "foo.go", Symbols: s3, Occurrences: o3}, 21 | } 22 | flattenedDocuments := SortDocuments(FlattenDocuments(documents)) 23 | 24 | expectedDocuments := []*Document{ 25 | {RelativePath: "bar.go", Symbols: s2, Occurrences: o2}, 26 | {RelativePath: "foo.go", Symbols: append(s1, s3...), Occurrences: append(o1, o3...)}, 27 | } 28 | if diff := cmp.Diff(expectedDocuments, flattenedDocuments, compareDocuments); diff != "" { 29 | t.Fatalf("unexpected documents (-want +got):\n%s", diff) 30 | } 31 | } 32 | 33 | var compareDocuments = cmp.Comparer(func(a, b *Document) bool { 34 | if a.Language != b.Language || a.RelativePath != b.RelativePath || a.Text != b.Text { 35 | return false 36 | } 37 | if !compareSlices(a.Symbols, b.Symbols, compareSymbol) { 38 | return false 39 | } 40 | if !compareSlices(a.Occurrences, b.Occurrences, compareOccurrence) { 41 | return false 42 | } 43 | 44 | return true 45 | }) 46 | 47 | func compareSlices[T any](as, bs []T, f func(a, b T) bool) bool { 48 | if len(as) != len(bs) { 49 | return false 50 | } 51 | 52 | for i, a := range as { 53 | if !f(a, bs[i]) { 54 | return false 55 | } 56 | } 57 | 58 | return true 59 | } 60 | 61 | func compareSymbol(a, b *SymbolInformation) bool { 62 | if a.Symbol != b.Symbol { 63 | return false 64 | } 65 | if a.Kind != b.Kind { 66 | return false 67 | } 68 | if a.DisplayName != b.DisplayName { 69 | return false 70 | } 71 | if a.SignatureDocumentation != b.SignatureDocumentation { 72 | return false 73 | } 74 | if a.EnclosingSymbol != b.EnclosingSymbol { 75 | return false 76 | } 77 | 78 | if !compareSlices(a.Documentation, b.Documentation, compareValue[string]) { 79 | return false 80 | } 81 | if !compareSlices(a.Relationships, b.Relationships, compareRelationship) { 82 | return false 83 | } 84 | 85 | return true 86 | } 87 | 88 | func compareOccurrence(a, b *Occurrence) bool { 89 | if a.Symbol != b.Symbol { 90 | return false 91 | } 92 | if a.SymbolRoles != b.SymbolRoles { 93 | return false 94 | } 95 | if a.SyntaxKind != b.SyntaxKind { 96 | return false 97 | } 98 | 99 | if !compareSlices(a.Range, b.Range, compareValue[int32]) { 100 | return false 101 | } 102 | if !compareSlices(a.OverrideDocumentation, b.OverrideDocumentation, compareValue[string]) { 103 | return false 104 | } 105 | if !compareSlices(a.Diagnostics, b.Diagnostics, compareDiagnostic) { 106 | return false 107 | } 108 | if !compareSlices(a.EnclosingRange, b.EnclosingRange, compareValue[int32]) { 109 | return false 110 | } 111 | 112 | return true 113 | } 114 | 115 | func compareRelationship(a, b *Relationship) bool { 116 | if a.Symbol != b.Symbol { 117 | return false 118 | } 119 | if a.IsReference != b.IsReference { 120 | return false 121 | } 122 | if a.IsImplementation != b.IsImplementation { 123 | return false 124 | } 125 | if a.IsTypeDefinition != b.IsTypeDefinition { 126 | return false 127 | } 128 | if a.IsDefinition != b.IsDefinition { 129 | return false 130 | } 131 | 132 | return true 133 | } 134 | 135 | func compareDiagnostic(a, b *Diagnostic) bool { 136 | if a.Severity != b.Severity { 137 | return false 138 | } 139 | if a.Code != b.Code { 140 | return false 141 | } 142 | if a.Message != b.Message { 143 | return false 144 | } 145 | if a.Source != b.Source { 146 | return false 147 | } 148 | if !compareSlices(a.Tags, b.Tags, compareValue[DiagnosticTag]) { 149 | return false 150 | } 151 | 152 | return true 153 | } 154 | 155 | func compareValue[T comparable](a, b T) bool { 156 | return a == b 157 | } 158 | -------------------------------------------------------------------------------- /bindings/go/scip/internal/compat_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "path/filepath" 7 | "sync/atomic" 8 | "testing" 9 | 10 | "github.com/sourcegraph/beaut" 11 | "github.com/sourcegraph/beaut/lib/knownwf" 12 | conciter "github.com/sourcegraph/conc/iter" 13 | "github.com/sourcegraph/scip/bindings/go/scip" 14 | "github.com/sourcegraph/scip/bindings/go/scip/internal/shared" 15 | "github.com/stretchr/testify/require" 16 | "google.golang.org/protobuf/proto" 17 | ) 18 | 19 | func TestParseCompat(t *testing.T) { 20 | for _, path := range shared.SampleIndexes() { 21 | t.Run(filepath.Base(path), func(t *testing.T) { 22 | t.Parallel() 23 | scipReader, err := os.Open(path) 24 | require.Nil(t, err) 25 | scipBytes, err := io.ReadAll(scipReader) 26 | require.Nil(t, err) 27 | scipIndex := scip.Index{} 28 | require.NoError(t, proto.Unmarshal(scipBytes, &scipIndex)) 29 | var total atomic.Int64 30 | conciter.ForEach(scipIndex.Documents, func(docPtr **scip.Document) { 31 | document := *docPtr 32 | if total.Load() > 1000*1000 { 33 | return 34 | } 35 | total.Add(int64(len(document.Occurrences))) 36 | var newSym scip.Symbol 37 | for i := 0; i < len(document.Occurrences); i++ { 38 | occ := document.Occurrences[i] 39 | oldSym, oldErr := ParsePartialSymbolV1ToBeDeleted(occ.Symbol, true) 40 | var newErr error 41 | require.NotPanics(t, func() { 42 | str := beaut.NewUTF8StringUnchecked(occ.Symbol, knownwf.UTF8DeserializedFromProtobufString) 43 | newErr = scip.ParseSymbolUTF8With(str, scip.ParseSymbolOptions{ 44 | IncludeDescriptors: true, 45 | RecordOutput: &newSym, 46 | }) 47 | }, "panic for symbol: %q", occ.Symbol) 48 | if oldErr != nil { 49 | require.Error(t, newErr, 50 | "old parser gave error %v but parse was successful with new parser (symbol: %q)", 51 | oldErr.Error(), occ.Symbol) 52 | continue 53 | } else if newErr != nil { 54 | require.NoError(t, newErr, 55 | "new parser gave error %v but parse was successful with old parser (symbol: %q)", 56 | newErr.Error(), occ.Symbol) 57 | } 58 | require.Equal(t, oldSym.Scheme, newSym.Scheme) 59 | require.Equal(t, oldSym.Package, newSym.Package) 60 | require.Equalf(t, len(oldSym.Descriptors), len(newSym.Descriptors), "symbol: %v, d1: %+v, d2: %+v", occ.Symbol, 61 | oldSym.Descriptors, newSym.Descriptors) 62 | for i, d := range oldSym.Descriptors { 63 | dnew := newSym.Descriptors[i] 64 | require.Equal(t, d.Name, dnew.Name, "symbol: %v", occ.Symbol) 65 | require.Equal(t, d.Suffix, dnew.Suffix, "symbol: %v", occ.Symbol) 66 | require.Equal(t, d.Disambiguator, dnew.Disambiguator, "symbol: %v", occ.Symbol) 67 | } 68 | } 69 | }) 70 | }) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /bindings/go/scip/internal/shared/sample_indexes.go: -------------------------------------------------------------------------------- 1 | package shared 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "path" 9 | "path/filepath" 10 | "slices" 11 | "strings" 12 | ) 13 | 14 | // SampleIndexes returns a list of paths to SCIP indexes for testing/benchmarking. 15 | func SampleIndexes() []string { 16 | workDir, err := os.Getwd() 17 | if err != nil { 18 | panic(fmt.Sprintf("failed to get working directory: %v", err)) 19 | } 20 | components := strings.Split(workDir, "/") 21 | fmt.Printf("") 22 | var dirEntries []os.DirEntry 23 | var indexesDir string 24 | var metadataPath string 25 | var allMetadata indexesMetadata 26 | for i := 0; i < len(components); i++ { 27 | if components[i] != "scip" { 28 | continue 29 | } 30 | indexesDir = filepath.Join("/", filepath.Join(components[:i+1]...), "dev", "sample_indexes") 31 | dirEntries, err = os.ReadDir(indexesDir) 32 | if err != nil { 33 | fmt.Fprintf(os.Stderr, "Could not locate sample indexes directory at: %v\n", err.Error()) 34 | continue 35 | } 36 | metadataPath = filepath.Join(indexesDir, "indexes-metadata.json") 37 | indexMetadataContents, err := os.ReadFile(metadataPath) 38 | if err != nil { 39 | panic(fmt.Sprintf("Failed to find metadata file for verifying SHAs: %s\n", err.Error())) 40 | } 41 | if err = json.Unmarshal(indexMetadataContents, &allMetadata); err != nil { 42 | panic(fmt.Sprintf("Failed to parse metadata file: %s (path: %q)\n", err.Error(), metadataPath)) 43 | } 44 | break 45 | } 46 | if len(dirEntries) == 0 { 47 | panic(fmt.Sprintf("could not locate sample indexes directory starting from parents of working directory: %q", workDir)) 48 | } 49 | out := []string{} 50 | for _, entry := range dirEntries { 51 | if strings.HasSuffix(entry.Name(), ".scip") { 52 | indexPath := path.Join(indexesDir, entry.Name()) 53 | if err := allMetadata.compareSHA(indexPath); err != nil { 54 | panic(err) 55 | } 56 | out = append(out, indexPath) 57 | } 58 | } 59 | return out 60 | } 61 | 62 | type indexesMetadata struct { 63 | Indexes []metadataEntry `json:"indexes"` 64 | } 65 | 66 | func (m *indexesMetadata) compareSHA(path string) error { 67 | filename := filepath.Base(path) 68 | idx := slices.IndexFunc(m.Indexes, func(m metadataEntry) bool { 69 | return m.Name == filename 70 | }) 71 | if idx == -1 { 72 | return fmt.Errorf("indexes-metadata.json missing entry for file %q", filename) 73 | } 74 | contents, err := os.ReadFile(path) 75 | if err != nil { 76 | return fmt.Errorf("failed to read index file at path %q: %w", path, err) 77 | } 78 | if got := computeSHA256(contents); got != m.Indexes[idx].SHA256 { 79 | return fmt.Errorf("index file SHA mismatch (-want, got):\n-%s (from indexes-metadata.json)\n+%s", 80 | m.Indexes[idx].SHA256, got) 81 | } 82 | return nil 83 | } 84 | 85 | type metadataEntry struct { 86 | Name string `json:"name"` 87 | SHA256 string `json:"sha256"` 88 | } 89 | 90 | func computeSHA256(contents []byte) string { 91 | hasher := sha256.New() 92 | hasher.Write(contents) 93 | return fmt.Sprintf("%x", hasher.Sum(nil)) 94 | } 95 | -------------------------------------------------------------------------------- /bindings/go/scip/internal/shared/shared.go: -------------------------------------------------------------------------------- 1 | // Package shared has functions used both by the old parser and the new parser. 2 | package shared 3 | 4 | func IsSimpleIdentifierCharacter(c rune) bool { 5 | return c == '_' || c == '+' || c == '-' || c == '$' || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || ('0' <= c && c <= '9') 6 | } 7 | 8 | // IsSimpleIdentifier matches the definition of in scip.proto. 9 | func IsSimpleIdentifier(s string) bool { 10 | for _, c := range s { 11 | if IsSimpleIdentifierCharacter(c) { 12 | continue 13 | } 14 | return false 15 | } 16 | return true 17 | } 18 | -------------------------------------------------------------------------------- /bindings/go/scip/memtest/DO_NOT_ADD_NEW_TEST_FILES_HERE: -------------------------------------------------------------------------------- 1 | This package should only have a single test 2 | so that we can use SetMemoryLimit without 3 | affecting other tests running in different 4 | goroutines in the same process. 5 | -------------------------------------------------------------------------------- /bindings/go/scip/memtest/low_mem_test.go: -------------------------------------------------------------------------------- 1 | package memtest 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "runtime" 9 | "runtime/debug" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/sourcegraph/beaut" 14 | "github.com/sourcegraph/beaut/lib/knownwf" 15 | "github.com/stretchr/testify/require" 16 | "google.golang.org/protobuf/proto" 17 | 18 | "github.com/sourcegraph/scip/bindings/go/scip" 19 | ) 20 | 21 | // Do not add any other tests in this sub-package, so that the 22 | // SetMemoryLimit call doesn't interfere with other running tests 23 | 24 | func TestLowMemoryParsing(t *testing.T) { 25 | // NOTE: This test must not run in parallel with the other one. 26 | tmpFile, err := os.CreateTemp(t.TempDir(), "very-large-index.scip") 27 | require.NoError(t, err) 28 | defer os.RemoveAll(tmpFile.Name()) 29 | 30 | // Total index size will be about (textSize + ε) * docCount ~= 128 MB 31 | const textSize = 128 * 1024 32 | const docCount = 1000 33 | { 34 | largeIndex := scip.Index{} 35 | for i := 0; i < docCount; i++ { 36 | doc := scip.Document{} 37 | doc.Text = strings.Repeat(fmt.Sprintf("%d", i), textSize) 38 | largeIndex.Documents = append(largeIndex.Documents, &doc) 39 | } 40 | indexBytes, err := proto.Marshal(&largeIndex) 41 | require.NoError(t, err) 42 | _, err = tmpFile.Write(indexBytes) 43 | require.NoError(t, err) 44 | 45 | _, err = tmpFile.Seek(0, io.SeekStart) 46 | require.NoError(t, err) 47 | 48 | runtime.GC() 49 | } 50 | 51 | require.Greater(t, docCount, 100) 52 | const maxDocsInMemory = docCount / 100 53 | debug.SetMemoryLimit(textSize * maxDocsInMemory) 54 | 55 | curDoc := &scip.Document{} 56 | indexVisitor := scip.IndexVisitor{ 57 | VisitDocument: func(_ context.Context, d *scip.Document) error { 58 | curDoc = d 59 | return nil 60 | }, 61 | } 62 | 63 | // No OOM 64 | err = indexVisitor.ParseStreaming(context.Background(), tmpFile) 65 | _ = curDoc 66 | require.NoError(t, err) 67 | } 68 | 69 | func TestParseSymbol_ZeroAllocationsIfMemoryAvailable(t *testing.T) { 70 | // This test must not run in parallel with the other one. 71 | str := beaut.NewUTF8StringUnchecked("cxx . mypkg v1.0.0 Foo#Bar(abcdef0123456789).", knownwf.UTF8OtherReason) 72 | sym := &scip.Symbol{ 73 | Scheme: "", 74 | Package: &scip.Package{}, 75 | Descriptors: []*scip.Descriptor{ 76 | &scip.Descriptor{}, 77 | &scip.Descriptor{}, 78 | }, 79 | } 80 | var before, after runtime.MemStats 81 | runtime.GC() 82 | old := debug.SetGCPercent(-1) 83 | 84 | runtime.ReadMemStats(&before) 85 | parseErr := scip.ParseSymbolUTF8With(str, scip.ParseSymbolOptions{IncludeDescriptors: true, RecordOutput: sym}) 86 | runtime.ReadMemStats(&after) 87 | 88 | require.NoError(t, parseErr) 89 | require.Equal(t, before.TotalAlloc, after.TotalAlloc, "pre-allocated symbol should not cause extra allocations") 90 | 91 | runtime.ReadMemStats(&before) 92 | validateErr := scip.ValidateSymbolUTF8(str) 93 | runtime.ReadMemStats(&after) 94 | 95 | require.NoError(t, validateErr) 96 | require.Equal(t, before.TotalAlloc, after.TotalAlloc, "pre-allocated symbol should not cause extra allocations") 97 | 98 | debug.SetGCPercent(old) 99 | } 100 | 101 | // Do not add any other tests in this sub-package, so that the 102 | // SetMemoryLimit call doesn't interfere with other running tests 103 | -------------------------------------------------------------------------------- /bindings/go/scip/parse.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/cockroachdb/errors" 8 | "google.golang.org/protobuf/encoding/protowire" 9 | "google.golang.org/protobuf/proto" 10 | ) 11 | 12 | // IndexVisitor is a struct of functions rather than an interface since Go 13 | // doesn't support adding new functions to interfaces with default 14 | // implementations, so adding new functions here for new fields in 15 | // the SCIP schema would break clients. Individual functions may be nil. 16 | type IndexVisitor struct { 17 | VisitMetadata func(ctx context.Context, m *Metadata) error 18 | VisitDocument func(ctx context.Context, d *Document) error 19 | VisitExternalSymbol func(ctx context.Context, si *SymbolInformation) error 20 | } 21 | 22 | // See https://protobuf.dev/programming-guides/encoding/#varints 23 | const maxVarintBytes = 10 24 | 25 | // ParseStreaming processes an index by incrementally reading input from the io.Reader. 26 | // 27 | // Parsing takes place at Document granularity for ease of use. 28 | func (pi *IndexVisitor) ParseStreaming(ctx context.Context, r io.Reader) error { 29 | // The tag is encoded as a varint with value: (field_number << 3) | wire_type 30 | // Varints < 128 fit in 1 byte, which means 4 bits are available for field 31 | // numbers. The Index type has less than 15 fields, so the tag will fit in 1 byte. 32 | tagBuf := make([]byte, 1) 33 | lenBuf := make([]byte, 0, maxVarintBytes) 34 | dataBuf := make([]byte, 0, 1024) 35 | 36 | for { 37 | numRead, err := r.Read(tagBuf) 38 | if err == io.EOF { 39 | return nil 40 | } 41 | if err != nil { 42 | return errors.Wrapf(err, "failed to read from index reader: %w") 43 | } 44 | if numRead == 0 { 45 | return errors.New("read 0 bytes from index") 46 | } 47 | fieldNumber, fieldType, errCode := protowire.ConsumeTag(tagBuf) 48 | if errCode < 0 { 49 | return errors.Wrap(protowire.ParseError(errCode), "failed to consume tag") 50 | } 51 | switch fieldNumber { 52 | // As per scip.proto, all of Metadata, Document and SymbolInformation are sub-messages 53 | case metadataFieldNumber, documentsFieldNumber, externalSymbolsFieldNumber: 54 | if fieldType != protowire.BytesType { 55 | return errors.Newf("expected LEN type tag for %s", indexFieldName(fieldNumber)) 56 | } 57 | lenBuf = lenBuf[:0] 58 | dataLenUint, err := readVarint(r, &lenBuf) 59 | dataLen := int(dataLenUint) 60 | if err != nil { 61 | return errors.Wrapf(err, "failed to read length for %s", indexFieldName(fieldNumber)) 62 | } 63 | if dataLen > cap(dataBuf) { 64 | dataBuf = make([]byte, dataLen) 65 | } else { 66 | dataBuf = dataBuf[:0] 67 | for i := 0; i < dataLen; i++ { 68 | dataBuf = append(dataBuf, 0) 69 | } 70 | } 71 | // Keep going when len == 0 instead of short-circuiting to preserve empty sub-messages 72 | if dataLen > 0 { 73 | numRead, err := io.ReadAtLeast(r, dataBuf, dataLen) 74 | if err != nil { 75 | return errors.Wrapf(err, "failed to read data for %s", indexFieldName(fieldNumber)) 76 | } 77 | if numRead != dataLen { 78 | return errors.Newf( 79 | "expected to read %d bytes based on LEN but read %d bytes", dataLen, numRead) 80 | } 81 | } 82 | if fieldNumber == metadataFieldNumber { 83 | if pi.VisitMetadata != nil { 84 | m := Metadata{} 85 | if err := proto.Unmarshal(dataBuf, &m); err != nil { 86 | return errors.Wrapf(err, "failed to read %s", indexFieldName(fieldNumber)) 87 | } 88 | if err := pi.VisitMetadata(ctx, &m); err != nil { 89 | return err 90 | } 91 | } 92 | } else if fieldNumber == documentsFieldNumber { 93 | if pi.VisitDocument != nil { 94 | d := Document{} 95 | if err := proto.Unmarshal(dataBuf, &d); err != nil { 96 | return errors.Wrapf(err, "failed to read %s", indexFieldName(fieldNumber)) 97 | } 98 | if err := pi.VisitDocument(ctx, &d); err != nil { 99 | return err 100 | } 101 | } 102 | } else if fieldNumber == externalSymbolsFieldNumber { 103 | if pi.VisitExternalSymbol != nil { 104 | s := SymbolInformation{} 105 | if err := proto.Unmarshal(dataBuf, &s); err != nil { 106 | return errors.Wrapf(err, "failed to read %s", indexFieldName(fieldNumber)) 107 | } 108 | if err := pi.VisitExternalSymbol(ctx, &s); err != nil { 109 | return err 110 | } 111 | } 112 | } else { 113 | return errors.Newf( 114 | "added new field with number: %v in scip.Index but missing unmarshaling code", fieldNumber) 115 | } 116 | default: 117 | return errors.Newf( 118 | "added new field with number: %v in scip.Index but forgot to update streaming parser", fieldNumber) 119 | } 120 | } 121 | } 122 | 123 | const ( 124 | metadataFieldNumber = 1 125 | documentsFieldNumber = 2 126 | externalSymbolsFieldNumber = 3 127 | ) 128 | 129 | // readVarint attempts to read a varint, using scratchBuf for temporary storage 130 | // 131 | // scratchBuf should be able to accommodate any varint size 132 | // based on its capacity, and be cleared before readVarint is called 133 | func readVarint(r io.Reader, scratchBuf *[]byte) (uint64, error) { 134 | nextByteBuf := make([]byte, 1, 1) 135 | for i := 0; i < cap(*scratchBuf); i++ { 136 | numRead, err := r.Read(nextByteBuf) 137 | if err != nil { 138 | return 0, errors.Wrapf(err, "failed to read %d-th byte of Varint. soFar: %v", i, scratchBuf) 139 | } 140 | if numRead == 0 { 141 | return 0, errors.Newf("failed to read %d-th byte of Varint. soFar: %v", scratchBuf) 142 | } 143 | nextByte := nextByteBuf[0] 144 | *scratchBuf = append(*scratchBuf, nextByte) 145 | if nextByte <= 127 { // https://protobuf.dev/programming-guides/encoding/#varints 146 | // Continuation bit is not set, so Varint must've ended 147 | break 148 | } 149 | } 150 | value, errCode := protowire.ConsumeVarint(*scratchBuf) 151 | if errCode < 0 { 152 | return value, protowire.ParseError(errCode) 153 | } 154 | return value, nil 155 | } 156 | 157 | func indexFieldName(i protowire.Number) string { 158 | if i == metadataFieldNumber { 159 | return "metadata" 160 | } else if i == documentsFieldNumber { 161 | return "documents" 162 | } else if i == externalSymbolsFieldNumber { 163 | return "external_symbols" 164 | } 165 | return "" 166 | } 167 | -------------------------------------------------------------------------------- /bindings/go/scip/parse_test.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "context" 7 | "io" 8 | "os" 9 | "regexp" 10 | "testing" 11 | 12 | "github.com/google/go-cmp/cmp" 13 | "github.com/google/gofuzz" 14 | "github.com/stretchr/testify/require" 15 | "google.golang.org/protobuf/encoding/protojson" 16 | "google.golang.org/protobuf/proto" 17 | ) 18 | 19 | func TestEmpty(t *testing.T) { 20 | index := Index{} 21 | checkRoundtrip(t, &index) 22 | } 23 | 24 | func TestFuzz(t *testing.T) { 25 | pat := regexp.MustCompile("^(state|sizeCache|unknownFields|SignatureDocumentation)$") 26 | f := fuzz.New().NumElements(0, 2).SkipFieldsWithPattern(pat) 27 | for i := 0; i < 100; i++ { 28 | index := Index{} 29 | f.Fuzz(&index) 30 | 31 | checkRoundtrip(t, &index) 32 | } 33 | } 34 | 35 | func getTestIndex(t *testing.T) *gzip.Reader { 36 | // Copied from the Sourcegraph monorepo, which triggered a bug 37 | // where Reader.read() didn't actually fill a buffer completely, 38 | // due to the presence of large documents. 39 | gzipped, err := os.Open("./testdata/index1.scip.gz") 40 | if err != nil { 41 | t.Fatalf("unexpected error reading test file: %s", err) 42 | } 43 | reader, err := gzip.NewReader(gzipped) 44 | if err != nil { 45 | t.Fatalf("unexpected error unzipping test file: %s", err) 46 | } 47 | return reader 48 | } 49 | 50 | func TestLargeDocuments(t *testing.T) { 51 | reader := getTestIndex(t) 52 | _ = parseStreaming(t, reader) 53 | } 54 | 55 | func TestDocumentsOnly(t *testing.T) { 56 | pat := regexp.MustCompile("^(state|sizeCache|unknownFields|SignatureDocumentation)$") 57 | f := fuzz.New().NumElements(0, 2).SkipFieldsWithPattern(pat) 58 | for i := 0; i < 100; i++ { 59 | index := Index{} 60 | f.Fuzz(&index) 61 | 62 | parsedIndex := Index{} 63 | 64 | indexVisitor := IndexVisitor{ 65 | VisitDocument: func(_ context.Context, document *Document) error { 66 | parsedIndex.Documents = append(parsedIndex.Documents, document) 67 | return nil 68 | }, 69 | } 70 | 71 | indexBytes, err := proto.Marshal(&index) 72 | require.NoError(t, err) 73 | bytesReader := bytes.NewReader(indexBytes) 74 | 75 | if err := indexVisitor.ParseStreaming(context.Background(), bytesReader); err != nil { 76 | t.Fatalf("got error parsing index %v", err) 77 | } 78 | 79 | onlyDocumentsIndex := Index{} 80 | onlyDocumentsIndex.Documents = index.Documents 81 | 82 | checkIndexEqual(t, &onlyDocumentsIndex, &parsedIndex) 83 | } 84 | } 85 | 86 | func checkIndexEqual(t *testing.T, expected *Index, got *Index) { 87 | if !proto.Equal(expected, got) { 88 | want := protojson.MarshalOptions{Multiline: true}.Format(expected) 89 | got := protojson.MarshalOptions{Multiline: true}.Format(got) 90 | diff := cmp.Diff(want, got) 91 | require.NotEqual(t, diff, "") 92 | t.Fatalf("index (-want, +got): %s", diff) 93 | } 94 | } 95 | 96 | func checkRoundtrip(t *testing.T, index *Index) { 97 | indexBytes, err := proto.Marshal(index) 98 | require.NoError(t, err) 99 | bytesReader := bytes.NewReader(indexBytes) 100 | 101 | parsedIndex := parseStreaming(t, bytesReader) 102 | 103 | checkIndexEqual(t, index, parsedIndex) 104 | } 105 | 106 | func parseStreaming(t *testing.T, reader io.Reader) *Index { 107 | parsedIndex := Index{} 108 | 109 | indexVisitor := IndexVisitor{ 110 | VisitMetadata: func(_ context.Context, metadata *Metadata) error { 111 | parsedIndex.Metadata = metadata 112 | return nil 113 | }, 114 | VisitDocument: func(_ context.Context, document *Document) error { 115 | parsedIndex.Documents = append(parsedIndex.Documents, document) 116 | return nil 117 | }, 118 | VisitExternalSymbol: func(_ context.Context, extSym *SymbolInformation) error { 119 | parsedIndex.ExternalSymbols = append(parsedIndex.ExternalSymbols, extSym) 120 | return nil 121 | }, 122 | } 123 | 124 | if err := indexVisitor.ParseStreaming(context.Background(), reader); err != nil { 125 | t.Fatalf("got error parsing index %v", err) 126 | } 127 | return &parsedIndex 128 | } 129 | -------------------------------------------------------------------------------- /bindings/go/scip/position.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | import "fmt" 4 | 5 | // Range represents [start, end) between two offset positions. 6 | // 7 | // NOTE: the github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol package 8 | // contains similarly shaped structs but this one exists primarily to make it 9 | // easier to work with SCIP encoded positions, which have the type []int32 10 | // in Protobuf payloads. 11 | type Range struct { 12 | Start Position 13 | End Position 14 | } 15 | 16 | // Position represents an offset position. 17 | type Position struct { 18 | Line int32 19 | Character int32 20 | } 21 | 22 | func (p Position) Compare(other Position) int { 23 | if p.Line < other.Line { 24 | return -1 25 | } 26 | if p.Line > other.Line { 27 | return 1 28 | } 29 | if p.Character < other.Character { 30 | return -1 31 | } 32 | if p.Character > other.Character { 33 | return 1 34 | } 35 | return 0 36 | } 37 | 38 | func (p Position) Less(other Position) bool { 39 | if p.Line < other.Line { 40 | return true 41 | } 42 | if p.Line > other.Line { 43 | return false 44 | } 45 | return p.Character < other.Character 46 | } 47 | 48 | //go:noinline 49 | func makeNewRangeError(startLine, endLine, startChar, endChar int32) (Range, error) { 50 | if startLine < 0 || endLine < 0 || startChar < 0 || endChar < 0 { 51 | return Range{}, NegativeOffsetsRangeError 52 | } 53 | if startLine > endLine || (startLine == endLine && startChar > endChar) { 54 | return Range{}, EndBeforeStartRangeError 55 | } 56 | panic("unreachable") 57 | } 58 | 59 | // NewRange constructs a Range while checking if the input is valid. 60 | func NewRange(scipRange []int32) (Range, error) { 61 | // N.B. This function is kept small so that it can be inlined easily. 62 | // See also: https://github.com/golang/go/issues/17566 63 | var startLine, endLine, startChar, endChar int32 64 | switch len(scipRange) { 65 | case 3: 66 | startLine = scipRange[0] 67 | endLine = startLine 68 | startChar = scipRange[1] 69 | endChar = scipRange[2] 70 | if startLine >= 0 && startChar >= 0 && endChar >= startChar { 71 | break 72 | } 73 | return makeNewRangeError(startLine, endLine, startChar, endChar) 74 | case 4: 75 | startLine = scipRange[0] 76 | startChar = scipRange[1] 77 | endLine = scipRange[2] 78 | endChar = scipRange[3] 79 | if startLine >= 0 && startChar >= 0 && 80 | ((endLine > startLine && endChar >= 0) || (endLine == startLine && endChar >= startChar)) { 81 | break 82 | } 83 | return makeNewRangeError(startLine, endLine, startChar, endChar) 84 | default: 85 | return Range{}, IncorrectLengthRangeError 86 | } 87 | return Range{Start: Position{Line: startLine, Character: startChar}, End: Position{Line: endLine, Character: endChar}}, nil 88 | } 89 | 90 | type RangeError int32 91 | 92 | const ( 93 | IncorrectLengthRangeError RangeError = iota 94 | NegativeOffsetsRangeError 95 | EndBeforeStartRangeError 96 | ) 97 | 98 | var _ error = RangeError(0) 99 | 100 | func (e RangeError) Error() string { 101 | switch e { 102 | case IncorrectLengthRangeError: 103 | return "incorrect length" 104 | case NegativeOffsetsRangeError: 105 | return "negative offsets" 106 | case EndBeforeStartRangeError: 107 | return "end before start" 108 | } 109 | panic("unhandled range error") 110 | } 111 | 112 | // NewRangeUnchecked converts an SCIP range into `Range` 113 | // 114 | // Pre-condition: The input slice must follow the SCIP range encoding. 115 | // https://sourcegraph.com/github.com/sourcegraph/scip/-/blob/scip.proto?L646:18-646:23 116 | func NewRangeUnchecked(scipRange []int32) Range { 117 | // Single-line case is most common 118 | endCharacter := scipRange[2] 119 | endLine := scipRange[0] 120 | if len(scipRange) == 4 { // multi-line 121 | endCharacter = scipRange[3] 122 | endLine = scipRange[2] 123 | } 124 | return Range{ 125 | Start: Position{ 126 | Line: scipRange[0], 127 | Character: scipRange[1], 128 | }, 129 | End: Position{ 130 | Line: endLine, 131 | Character: endCharacter, 132 | }, 133 | } 134 | } 135 | 136 | func (r Range) IsSingleLine() bool { 137 | return r.Start.Line == r.End.Line 138 | } 139 | 140 | func (r Range) SCIPRange() []int32 { 141 | if r.Start.Line == r.End.Line { 142 | return []int32{r.Start.Line, r.Start.Character, r.End.Character} 143 | } 144 | return []int32{r.Start.Line, r.Start.Character, r.End.Line, r.End.Character} 145 | } 146 | 147 | // Contains checks if position is within the range 148 | func (r Range) Contains(position Position) bool { 149 | return !position.Less(r.Start) && position.Less(r.End) 150 | } 151 | 152 | // Intersects checks if two ranges intersect. 153 | // 154 | // case 1: r1.Start >= other.Start && r1.Start < other.End 155 | // case 2: r2.Start >= r1.Start && r2.Start < r1.End 156 | func (r Range) Intersects(other Range) bool { 157 | return r.Start.Less(other.End) && other.Start.Less(r.End) 158 | } 159 | 160 | // Compare compares two ranges. 161 | // 162 | // Returns 0 if the ranges intersect (not just if they're equal). 163 | func (r Range) Compare(other Range) int { 164 | if r.Intersects(other) { 165 | return 0 166 | } 167 | return r.Start.Compare(other.Start) 168 | } 169 | 170 | // Less compares two ranges, consistent with Compare. 171 | // 172 | // r.Compare(other) < 0 iff r.Less(other). 173 | func (r Range) Less(other Range) bool { 174 | return r.End.Compare(other.Start) <= 0 175 | } 176 | 177 | // CompareStrict compares two ranges. 178 | // 179 | // Returns 0 iff the ranges are exactly equal. 180 | func (r Range) CompareStrict(other Range) int { 181 | if ret := r.Start.Compare(other.Start); ret != 0 { 182 | return ret 183 | } 184 | return r.End.Compare(other.End) 185 | } 186 | 187 | // LessStrict compares two ranges, consistent with CompareStrict. 188 | // 189 | // r.CompareStrict(other) < 0 iff r.LessStrict(other). 190 | func (r Range) LessStrict(other Range) bool { 191 | if ret := r.Start.Compare(other.Start); ret != 0 { 192 | return ret < 0 193 | } 194 | return r.End.Less(other.End) 195 | } 196 | 197 | func (r Range) String() string { 198 | return fmt.Sprintf("%d:%d-%d:%d", r.Start.Line, r.Start.Character, r.End.Line, r.End.Character) 199 | } 200 | -------------------------------------------------------------------------------- /bindings/go/scip/sanitize.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | import ( 4 | "unicode/utf8" 5 | ) 6 | 7 | // SanitizeDocument ensures that all strings in the given document are valid UTF-8. 8 | // This is a requirement for successful protobuf encoding. 9 | func SanitizeDocument(document *Document) *Document { 10 | document.Language = sanitizeString(document.Language) 11 | document.RelativePath = sanitizeString(document.RelativePath) 12 | document.Occurrences = SanitizeOccurrences(document.Occurrences) 13 | document.Symbols = SanitizeSymbols(document.Symbols) 14 | return document 15 | } 16 | 17 | // SanitizeOccurrences ensures that all strings in the given occurrence slice are valid UTF-8. 18 | // The input slice is modified in-place but returned for convenience. 19 | // This is a requirement for successful protobuf encoding. 20 | func SanitizeOccurrences(occurrences []*Occurrence) []*Occurrence { 21 | for i, occurrence := range occurrences { 22 | occurrences[i] = SanitizeOccurrence(occurrence) 23 | } 24 | 25 | return occurrences 26 | } 27 | 28 | // SanitizeOccurrence ensures that all strings in the given occurrence are valid UTF-8. 29 | // This is a requirement for successful protobuf encoding. 30 | func SanitizeOccurrence(occurrence *Occurrence) *Occurrence { 31 | occurrence.Symbol = sanitizeString(occurrence.Symbol) 32 | occurrence.OverrideDocumentation = sanitizeStringSlice(occurrence.OverrideDocumentation) 33 | occurrence.Diagnostics = SanitizeDiagnostics(occurrence.Diagnostics) 34 | return occurrence 35 | } 36 | 37 | // SanitizeDiagnostics ensures that all strings in the given diagnostic slice are valid UTF-8. 38 | // The input slice is modified in-place but returned for convenience. 39 | // This is a requirement for successful protobuf encoding. 40 | func SanitizeDiagnostics(diagnostics []*Diagnostic) []*Diagnostic { 41 | for i, diagnostic := range diagnostics { 42 | diagnostics[i] = SanitizeDiagnostic(diagnostic) 43 | } 44 | 45 | return diagnostics 46 | } 47 | 48 | // SanitizeDiagnostic ensures that all strings in the given diagnostic are valid UTF-8. 49 | // This is a requirement for successful protobuf encoding. 50 | func SanitizeDiagnostic(diagnostic *Diagnostic) *Diagnostic { 51 | diagnostic.Code = sanitizeString(diagnostic.Code) 52 | diagnostic.Message = sanitizeString(diagnostic.Message) 53 | diagnostic.Source = sanitizeString(diagnostic.Source) 54 | return diagnostic 55 | } 56 | 57 | // SanitizeSymbols ensures that all strings in the given symbols slice are valid UTF-8. 58 | // The input slice is modified in-place but returned for convenience. 59 | // This is a requirement for successful protobuf encoding. 60 | func SanitizeSymbols(symbols []*SymbolInformation) []*SymbolInformation { 61 | for i, symbol := range symbols { 62 | symbols[i] = SanitizeSymbol(symbol) 63 | } 64 | 65 | return symbols 66 | } 67 | 68 | // SanitizeSymbol ensures that all strings in the given symbol are valid UTF-8. 69 | // This is a requirement for successful protobuf encoding. 70 | func SanitizeSymbol(symbol *SymbolInformation) *SymbolInformation { 71 | symbol.Symbol = sanitizeString(symbol.Symbol) 72 | symbol.Documentation = sanitizeStringSlice(symbol.Documentation) 73 | 74 | for _, relationship := range symbol.Relationships { 75 | relationship.Symbol = sanitizeString(relationship.Symbol) 76 | } 77 | 78 | return symbol 79 | } 80 | 81 | // sanitizeStringSlice ensures the strings in the given slice are all valid UTF-8. 82 | // The input slice is modified in-place but returned for convenience. 83 | // This is a requirement for successful protobuf encoding. 84 | func sanitizeStringSlice(ss []string) []string { 85 | for i, s := range ss { 86 | ss[i] = sanitizeString(s) 87 | } 88 | 89 | return ss 90 | } 91 | 92 | // sanitizeString coerces a string into valid UTF-8 (if it's not already). 93 | func sanitizeString(s string) string { 94 | if utf8.ValidString(s) { 95 | return s 96 | } 97 | 98 | // This seems redundant, but it isn't: it magically makes the string valid UTF-8 99 | return string([]rune(s)) 100 | } 101 | -------------------------------------------------------------------------------- /bindings/go/scip/sort_test.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | import ( 4 | "slices" 5 | "testing" 6 | 7 | "github.com/google/go-cmp/cmp" 8 | "github.com/stretchr/testify/require" 9 | "pgregory.net/rapid" 10 | ) 11 | 12 | func TestFindOccurrences(t *testing.T) { 13 | occurrences := []*Occurrence{ 14 | {Range: []int32{0, 3, 4, 5}}, 15 | {Range: []int32{1, 3, 3, 5}}, 16 | {Range: []int32{2, 3, 5}}, 17 | {Range: []int32{5, 3, 5}}, 18 | {Range: []int32{6, 3, 5}}, 19 | } 20 | 21 | var matchingRanges [][]int32 22 | for _, occurrence := range FindOccurrences(occurrences, 2, 4) { 23 | matchingRanges = append(matchingRanges, occurrence.Range) 24 | } 25 | 26 | expected := [][]int32{ 27 | occurrences[2].Range, 28 | occurrences[1].Range, 29 | occurrences[0].Range, 30 | } 31 | if diff := cmp.Diff(expected, matchingRanges); diff != "" { 32 | t.Errorf("unexpected FindOccurrences result (-want +got):\n%s", diff) 33 | } 34 | } 35 | 36 | func TestSortOccurrences(t *testing.T) { 37 | occurrences := []*Occurrence{ 38 | {Range: []int32{2, 3, 5}}, // rank 2 39 | {Range: []int32{11, 10, 12}}, // rank 10 40 | {Range: []int32{6, 3, 5}}, // rank 4 41 | {Range: []int32{10, 4, 8}}, // rank 6 42 | {Range: []int32{10, 10, 12}}, // rank 7 43 | {Range: []int32{0, 3, 4, 5}}, // rank 0 44 | {Range: []int32{12, 1, 13, 12}}, // rank 11 45 | {Range: []int32{11, 1, 3}}, // rank 8 46 | {Range: []int32{5, 3, 5}}, // rank 3 47 | {Range: []int32{10, 1, 3}}, // rank 5 48 | {Range: []int32{12, 10, 13, 3}}, // rank 13 49 | {Range: []int32{11, 4, 8}}, // rank 9 50 | {Range: []int32{12, 4, 13, 8}}, // rank 12 51 | {Range: []int32{1, 3, 3, 5}}, // rank 1 52 | } 53 | unsorted := make([]*Occurrence, len(occurrences)) 54 | copy(unsorted, occurrences) 55 | 56 | ranges := [][]int32{} 57 | for _, r := range SortOccurrences(unsorted) { 58 | ranges = append(ranges, r.Range) 59 | } 60 | 61 | expected := [][]int32{ 62 | occurrences[5].Range, 63 | occurrences[13].Range, 64 | occurrences[0].Range, 65 | occurrences[8].Range, 66 | occurrences[2].Range, 67 | occurrences[9].Range, 68 | occurrences[3].Range, 69 | occurrences[4].Range, 70 | occurrences[7].Range, 71 | occurrences[11].Range, 72 | occurrences[1].Range, 73 | occurrences[6].Range, 74 | occurrences[12].Range, 75 | occurrences[10].Range, 76 | } 77 | if diff := cmp.Diff(expected, ranges); diff != "" { 78 | t.Errorf("unexpected occurrence order (-want +got):\n%s", diff) 79 | } 80 | } 81 | 82 | func TestSortRanges(t *testing.T) { 83 | occurrences := []Range{ 84 | NewRangeUnchecked([]int32{2, 3, 5}), // rank 2 85 | NewRangeUnchecked([]int32{11, 10, 12}), // rank 10 86 | NewRangeUnchecked([]int32{6, 3, 5}), // rank 4 87 | NewRangeUnchecked([]int32{10, 4, 8}), // rank 6 88 | NewRangeUnchecked([]int32{10, 10, 12}), // rank 7 89 | NewRangeUnchecked([]int32{0, 3, 4, 5}), // rank 0 90 | NewRangeUnchecked([]int32{12, 1, 13, 12}), // rank 11 91 | NewRangeUnchecked([]int32{11, 1, 3}), // rank 8 92 | NewRangeUnchecked([]int32{5, 3, 5}), // rank 3 93 | NewRangeUnchecked([]int32{10, 1, 3}), // rank 5 94 | NewRangeUnchecked([]int32{12, 10, 13, 3}), // rank 13 95 | NewRangeUnchecked([]int32{11, 4, 8}), // rank 9 96 | NewRangeUnchecked([]int32{12, 4, 13, 8}), // rank 12 97 | NewRangeUnchecked([]int32{1, 3, 3, 5}), // rank 1 98 | } 99 | unsorted := make([]Range, len(occurrences)) 100 | copy(unsorted, occurrences) 101 | 102 | ranges := [][]int32{} 103 | for _, r := range SortRanges(unsorted) { 104 | ranges = append(ranges, r.SCIPRange()) 105 | } 106 | 107 | // TODO - better data 108 | expected := [][]int32{ 109 | occurrences[5].SCIPRange(), 110 | occurrences[13].SCIPRange(), 111 | occurrences[0].SCIPRange(), 112 | occurrences[8].SCIPRange(), 113 | occurrences[2].SCIPRange(), 114 | occurrences[9].SCIPRange(), 115 | occurrences[3].SCIPRange(), 116 | occurrences[4].SCIPRange(), 117 | occurrences[7].SCIPRange(), 118 | occurrences[11].SCIPRange(), 119 | occurrences[1].SCIPRange(), 120 | occurrences[6].SCIPRange(), 121 | occurrences[12].SCIPRange(), 122 | occurrences[10].SCIPRange(), 123 | } 124 | if diff := cmp.Diff(expected, ranges); diff != "" { 125 | t.Errorf("unexpected occurrence order (-want +got):\n%s", diff) 126 | } 127 | } 128 | 129 | func genSymbolInfo() *rapid.Generator[*SymbolInformation] { 130 | return rapid.Custom(func(t *rapid.T) *SymbolInformation { 131 | return &SymbolInformation{Symbol: rapid.String().Draw(t, "symbol")} 132 | }) 133 | } 134 | 135 | func TestFindSymbolBinarySearch(t *testing.T) { 136 | rapid.Check(t, func(t *rapid.T) { 137 | symbolInfoGen := genSymbolInfo() 138 | symbolInfos := rapid.SliceOfN(symbolInfoGen, 0, 10).Draw(t, "symbolInfos") 139 | doc := &Document{Symbols: symbolInfos} 140 | canonicalDoc := CanonicalizeDocument(doc) 141 | for _, symbolInfo := range symbolInfos { 142 | got := FindSymbolBinarySearch(canonicalDoc, symbolInfo.Symbol) 143 | require.NotNil(t, got) 144 | require.Equal(t, symbolInfo.Symbol, got.Symbol) 145 | } 146 | other := rapid.String().Draw(t, "otherSymbol") 147 | isInOriginalSlice := slices.ContainsFunc(symbolInfos, func(info *SymbolInformation) bool { 148 | return info.Symbol == other 149 | }) 150 | got := FindSymbolBinarySearch(canonicalDoc, other) 151 | if isInOriginalSlice { 152 | require.NotNil(t, got) 153 | require.Equal(t, other, got.Symbol) 154 | } else { 155 | require.Nil(t, got) 156 | } 157 | }) 158 | } 159 | -------------------------------------------------------------------------------- /bindings/go/scip/source_file.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | import ( 4 | "encoding/json" 5 | "io/fs" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | ) 10 | 11 | // SourceFile includes helper methods to deal with source files. 12 | type SourceFile struct { 13 | AbsolutePath string 14 | RelativePath string 15 | Text string 16 | Lines []string 17 | } 18 | 19 | func NewSourceFile(absolutePath, relativePath, code string) *SourceFile { 20 | return &SourceFile{ 21 | AbsolutePath: absolutePath, 22 | RelativePath: relativePath, 23 | Text: code, 24 | Lines: strings.Split(code, "\n"), 25 | } 26 | } 27 | 28 | // NewSourcesFromDirectory recursively walks the provided directory and creates a SourceFile for every regular file. 29 | func NewSourcesFromDirectory(directory string) ([]*SourceFile, error) { 30 | var result []*SourceFile 31 | err := filepath.Walk(directory, func(path string, info fs.FileInfo, err error) error { 32 | if err != nil { 33 | return err 34 | } 35 | if info.IsDir() { 36 | return nil 37 | } 38 | relativePath, err := filepath.Rel(directory, path) 39 | if err != nil { 40 | return err 41 | } 42 | sourceFile, err := NewSourceFileFromPath(path, relativePath) 43 | if err != nil { 44 | return err 45 | } 46 | result = append(result, sourceFile) 47 | return nil 48 | }) 49 | return result, err 50 | } 51 | 52 | // NewSourceFileFromPath reads the provided absolute path from disk and returns a SourceFile. 53 | func NewSourceFileFromPath(absolutePath, relativePath string) (*SourceFile, error) { 54 | text, err := os.ReadFile(absolutePath) 55 | if err != nil { 56 | return nil, err 57 | } 58 | return NewSourceFile(absolutePath, relativePath, string(text)), nil 59 | } 60 | 61 | func (d *SourceFile) String() string { 62 | data, err := json.Marshal(&d) 63 | if err != nil { 64 | panic(err) 65 | } 66 | return string(data) 67 | } 68 | 69 | // RangeText returns the substring of the source file contents that enclose the provided range. 70 | func (d *SourceFile) RangeText(position Range) string { 71 | result := strings.Builder{} 72 | for line := position.Start.Line; line <= position.End.Line; line++ { 73 | start := position.Start.Character 74 | if line > position.Start.Line { 75 | result.WriteString("\n") 76 | start = 0 77 | } 78 | end := position.End.Character 79 | if line < position.End.Line { 80 | end = int32(len(d.Lines[line])) 81 | } 82 | result.WriteString(d.Lines[line][start:end]) 83 | } 84 | return result.String() 85 | } 86 | -------------------------------------------------------------------------------- /bindings/go/scip/source_file_test.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/hexops/autogold/v2" 8 | ) 9 | 10 | const testFileContents = `Mary had a little lamb, 11 | Its fleece was white as snow. 12 | And everywhere that Mary went, 13 | The lamb was sure to go. 14 | ` 15 | 16 | func TestRangeText(t *testing.T) { 17 | // Update expect values with `go test ./bindings/go/scip -update` 18 | testCases := []struct { 19 | range_ []int32 20 | expect autogold.Value 21 | }{ 22 | {range_: []int32{0, 0, 0, 0}, expect: autogold.Expect("")}, 23 | {range_: []int32{0, 0, 0, 8}, expect: autogold.Expect("Mary had")}, 24 | {range_: []int32{0, 0, 1, 0}, expect: autogold.Expect("Mary had a little lamb,\n")}, 25 | {range_: []int32{0, 0, 1, 10}, expect: autogold.Expect("Mary had a little lamb,\nIts fleece")}, 26 | {range_: []int32{0, 0, 4, 0}, expect: autogold.Expect(`Mary had a little lamb, 27 | Its fleece was white as snow. 28 | And everywhere that Mary went, 29 | The lamb was sure to go. 30 | `)}, 31 | } 32 | 33 | sourceFile := NewSourceFile("", "", testFileContents) 34 | 35 | for _, testCase := range testCases { 36 | r := testCase.range_ 37 | range_ := Range{Start: Position{Line: r[0], Character: r[1]}, End: Position{Line: r[2], Character: r[3]}} 38 | t.Run(fmt.Sprintf("%v", range_), func(t *testing.T) { 39 | testCase.expect.Equal(t, sourceFile.RangeText(range_)) 40 | }) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /bindings/go/scip/symbol.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/sourcegraph/beaut" 7 | ) 8 | 9 | // IsGlobalSymbol returns true if the symbol is obviously not a local symbol. 10 | // 11 | // CAUTION: Does not perform full validation of the symbol string's contents. 12 | func IsGlobalSymbol(symbol string) bool { 13 | return !IsLocalSymbol(symbol) 14 | } 15 | 16 | // IsLocalSymbol returns true if the symbol is obviously not a global symbol. 17 | // 18 | // CAUTION: Does not perform full validation of the symbol string's contents. 19 | func IsLocalSymbol(symbol string) bool { 20 | return strings.HasPrefix(symbol, "local ") 21 | } 22 | 23 | // ParseSymbol parses an SCIP string into the Symbol message. 24 | // 25 | // Prefer using ParseSymbolUTF8 for strings already known to 26 | // be valid UTF-8 encoded strings. For example, the Protobuf 27 | // string type is guaranteed to be converted to a UTF-8 string 28 | // in Go, you can avoid an extra UTF-8 validity check with 29 | // this API. 30 | func ParseSymbol(symbol string) (*Symbol, error) { 31 | s, err := beaut.NewUTF8String(symbol) 32 | if err != nil { 33 | return nil, err 34 | } 35 | return ParseSymbolUTF8(s) 36 | } 37 | 38 | // ParseSymbolUTF8 parses an SCIP string into the Symbol message. 39 | // 40 | // Unlike ParseSymbol, this skips UTF-8 validation. To customize 41 | // parsing behavior, use ParseSymbolUTF8With. 42 | func ParseSymbolUTF8(symbol beaut.UTF8String) (*Symbol, error) { 43 | s := Symbol{} 44 | err := ParseSymbolUTF8With(symbol, ParseSymbolOptions{IncludeDescriptors: true, RecordOutput: &s}) 45 | return &s, err 46 | } 47 | 48 | type emptySymbolError struct{} 49 | 50 | func (emptySymbolError) Error() string { 51 | return "empty symbol" 52 | } 53 | 54 | type ParseSymbolOptions struct { 55 | // IncludeDescriptors indicates whether the the descriptors in the symbol should also be 56 | // parsed. This setting is only respected for global symbols. 57 | IncludeDescriptors bool 58 | // RecordOutput, if provided, is used to record the output of the symbol parser. 59 | // 60 | // Otherwise, only validation is performed without any extraneous heap 61 | // allocations (except for optimization misses by the Go compiler). 62 | RecordOutput *Symbol 63 | } 64 | 65 | func ParseSymbolUTF8With(symbol beaut.UTF8String, options ParseSymbolOptions) error { 66 | if symbol.Len() == 0 { 67 | return emptySymbolError{} 68 | } 69 | return parseSymbolV2(symbol, options.IncludeDescriptors, options.RecordOutput) 70 | } 71 | 72 | func ValidateSymbolUTF8(symbol beaut.UTF8String) error { 73 | return ParseSymbolUTF8With(symbol, ParseSymbolOptions{IncludeDescriptors: true, RecordOutput: nil}) 74 | } 75 | -------------------------------------------------------------------------------- /bindings/go/scip/symbol_formatter_test.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestSymbolFormatter(t *testing.T) { 11 | expected := "zeroth escape first escape second escape third escape `github.com/foo/``bar/n2/n3/n4`/T#f()." 12 | symbol := &Symbol{ 13 | Scheme: "zeroth escape", 14 | Package: &Package{ 15 | Manager: "first escape", 16 | Name: "second escape", 17 | Version: "third escape", 18 | }, 19 | Descriptors: []*Descriptor{ 20 | {Name: "github.com/foo/`bar/n2/n3/n4", Suffix: Descriptor_Namespace}, 21 | {Name: "T", Suffix: Descriptor_Type}, 22 | {Name: "f", Suffix: Descriptor_Method}, 23 | }, 24 | } 25 | 26 | if diff := cmp.Diff(expected, VerboseSymbolFormatter.FormatSymbol(symbol)); diff != "" { 27 | t.Fatalf("unexpected response (-want +got):\n%s", diff) 28 | } 29 | } 30 | 31 | func TestSymbolFormatterRoundTrip(t *testing.T) { 32 | type test struct { 33 | Symbol string 34 | } 35 | tests := []test{ 36 | {"lsif-java maven package 1.0.0 java/io/File#Entry.method(+1).(param)[TypeParam]"}, 37 | {"rust-analyzer cargo std 1.0.0 macros/println!"}, 38 | {"zeroth escape first escape second escape third escape `github.com/foo/``bar/n2/n3/n4`/T#f()."}, 39 | } 40 | for _, test := range tests { 41 | t.Run(test.Symbol, func(t *testing.T) { 42 | formatted, err := VerboseSymbolFormatter.Format(test.Symbol) 43 | require.Nil(t, err) 44 | 45 | if diff := cmp.Diff(test.Symbol, formatted); diff != "" { 46 | t.Fatalf("unexpected response (-want +got):\n%s", diff) 47 | } 48 | }) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /bindings/go/scip/symbol_role.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | func (r SymbolRole) Matches(occ *Occurrence) bool { 4 | return occ.SymbolRoles&int32(r) > 0 5 | } -------------------------------------------------------------------------------- /bindings/go/scip/symbol_table.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | // SymbolTable returns a map of SymbolInformation values keyed by the symbol field. 4 | func (x *Document) SymbolTable() map[string]*SymbolInformation { 5 | symtab := map[string]*SymbolInformation{} 6 | for _, info := range x.Symbols { 7 | symtab[info.Symbol] = info 8 | } 9 | return symtab 10 | } 11 | -------------------------------------------------------------------------------- /bindings/go/scip/symbol_test.go: -------------------------------------------------------------------------------- 1 | package scip 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestParseSymbol(t *testing.T) { 11 | type test struct { 12 | Symbol string 13 | Expected *Symbol 14 | } 15 | tests := []test{ 16 | {Symbol: "local a", Expected: &Symbol{ 17 | Scheme: "local", 18 | Descriptors: []*Descriptor{{Name: "a", Suffix: Descriptor_Local}}, 19 | }}, 20 | {Symbol: "a b c d method().", Expected: &Symbol{ 21 | Scheme: "a", 22 | Package: &Package{ 23 | Manager: "b", 24 | Name: "c", 25 | Version: "d", 26 | }, 27 | Descriptors: []*Descriptor{{Name: "method", Suffix: Descriptor_Method}}, 28 | }}, 29 | // Backtick-escaped descriptor 30 | {Symbol: "a b c d `e f`.", Expected: &Symbol{ 31 | Scheme: "a", 32 | Package: &Package{ 33 | Manager: "b", 34 | Name: "c", 35 | Version: "d", 36 | }, 37 | Descriptors: []*Descriptor{{Name: "e f", Suffix: Descriptor_Term}}, 38 | }}, 39 | // Space-escaped package name 40 | {Symbol: "a b c d e f.", Expected: &Symbol{ 41 | Scheme: "a", 42 | Package: &Package{ 43 | Manager: "b c", 44 | Name: "d", 45 | Version: "e", 46 | }, 47 | Descriptors: []*Descriptor{{Name: "f", Suffix: Descriptor_Term}}, 48 | }}, 49 | { 50 | Symbol: "lsif-java maven package 1.0.0 java/io/File#Entry.method(+1).(param)[TypeParam]", 51 | Expected: &Symbol{ 52 | Scheme: "lsif-java", 53 | Package: &Package{Manager: "maven", Name: "package", Version: "1.0.0"}, 54 | Descriptors: []*Descriptor{ 55 | {Name: "java", Suffix: Descriptor_Namespace}, 56 | {Name: "io", Suffix: Descriptor_Namespace}, 57 | {Name: "File", Suffix: Descriptor_Type}, 58 | {Name: "Entry", Suffix: Descriptor_Term}, 59 | {Name: "method", Disambiguator: "+1", Suffix: Descriptor_Method}, 60 | {Name: "param", Suffix: Descriptor_Parameter}, 61 | {Name: "TypeParam", Suffix: Descriptor_TypeParameter}, 62 | }, 63 | }, 64 | }, 65 | { 66 | Symbol: "rust-analyzer cargo std 1.0.0 macros/println!", 67 | Expected: &Symbol{ 68 | Scheme: "rust-analyzer", 69 | Package: &Package{Manager: "cargo", Name: "std", Version: "1.0.0"}, 70 | Descriptors: []*Descriptor{ 71 | {Name: "macros", Suffix: Descriptor_Namespace}, 72 | {Name: "println", Suffix: Descriptor_Macro}, 73 | }, 74 | }, 75 | }, 76 | { 77 | Symbol: "cxx . todo-pkg todo-version gfx/Rect#x(455f465bc33b4cdf).", 78 | Expected: &Symbol{ 79 | Scheme: "cxx", 80 | Package: &Package{Name: "todo-pkg", Version: "todo-version"}, 81 | Descriptors: []*Descriptor{ 82 | {Name: "gfx", Suffix: Descriptor_Namespace}, 83 | {Name: "Rect", Suffix: Descriptor_Type}, 84 | {Name: "x", Disambiguator: "455f465bc33b4cdf", Suffix: Descriptor_Method}, 85 | }, 86 | }, 87 | }, 88 | { 89 | Symbol: "cxx . . $ `/4398592474888995393/assert.h:92:11`!", 90 | Expected: &Symbol{ 91 | Scheme: "cxx", 92 | Package: &Package{Manager: "", Name: "", Version: "$"}, 93 | Descriptors: []*Descriptor{ 94 | {Name: "/4398592474888995393/assert.h:92:11", Suffix: Descriptor_Macro}, 95 | }, 96 | }, 97 | }, 98 | { 99 | Symbol: "cxx . . $ llvm/itanium_demangle/OutputBuffer#empty(50ce9a9e25b4a850).", 100 | Expected: &Symbol{ 101 | Scheme: "cxx", 102 | Package: &Package{Manager: "", Name: "", Version: "$"}, 103 | Descriptors: []*Descriptor{ 104 | {Name: "llvm", Suffix: Descriptor_Namespace}, 105 | {Name: "itanium_demangle", Suffix: Descriptor_Namespace}, 106 | {Name: "OutputBuffer", Suffix: Descriptor_Type}, 107 | {Name: "empty", Disambiguator: "50ce9a9e25b4a850", Suffix: Descriptor_Method}, 108 | }, 109 | }, 110 | }, 111 | } 112 | for _, test := range tests { 113 | t.Run(test.Symbol, func(t *testing.T) { 114 | obtained, err := ParseSymbol(test.Symbol) 115 | require.Nil(t, err) 116 | if diff := cmp.Diff(test.Expected.String(), obtained.String()); diff != "" { 117 | t.Fatalf("unexpected response (-want +got):\n%s", diff) 118 | } 119 | }) 120 | } 121 | } 122 | 123 | func TestParseSymbolError(t *testing.T) { 124 | for _, symbolName := range []string{ 125 | "", 126 | "lsif-java maven package 1.0.0", 127 | "lsif-java maven package 1.0.0 java/io/File#Entry.trailingstring", 128 | "lsif-java maven package 1.0.0 java/io/File#Entry.unrecognizedSuffix@", 129 | "lsif-java maven package 1.0.0 java/io/File#Entry.nonSimpλeIdentifier.", 130 | "local 🧠", 131 | "local ", 132 | "local &&&", 133 | } { 134 | require.NotPanics(t, func() { 135 | if _, err := ParseSymbol(symbolName); err == nil { 136 | t.Fatalf("expected error from parsing %q", symbolName) 137 | } 138 | }, "panic when parsing %q", symbolName) 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /bindings/go/scip/testdata/index1.scip.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sourcegraph/scip/0c0c2b8fcb04fa05025967daad2df767b73061b3/bindings/go/scip/testdata/index1.scip.gz -------------------------------------------------------------------------------- /bindings/go/scip/testutil/format_test.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestIsScipRangeLess(t *testing.T) { 10 | testCases := []struct { 11 | lhs []int32 12 | rhs []int32 13 | }{ 14 | {[]int32{0, 1, 2}, []int32{1, 0, 0}}, 15 | {[]int32{0, 1, 10}, []int32{0, 2, 3}}, 16 | {[]int32{0, 1, 2, 3}, []int32{0, 2, 4}}, 17 | {[]int32{0, 1, 4}, []int32{0, 1, 2, 3}}, 18 | {[]int32{0, 1, 2, 3}, []int32{0, 1, 4, 0}}, 19 | {[]int32{0, 1, 2, 3}, []int32{0, 1, 2, 4}}, 20 | } 21 | for _, testCase := range testCases { 22 | require.Truef(t, isSCIPRangeLess(testCase.lhs, testCase.rhs), 23 | "%v ≮ %v", testCase.lhs, testCase.rhs) 24 | require.Falsef(t, isSCIPRangeLess(testCase.rhs, testCase.lhs), 25 | "%v < %v", testCase.rhs, testCase.lhs) 26 | 27 | require.Falsef(t, isSCIPRangeLess(testCase.lhs, testCase.lhs), 28 | "%v < %v", testCase.lhs, testCase.lhs) 29 | require.Falsef(t, isSCIPRangeLess(testCase.lhs, testCase.lhs), 30 | "%v < %v", testCase.rhs, testCase.rhs) 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /bindings/go/scip/testutil/snapshot_testing.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | 10 | "github.com/hexops/gotextdiff" 11 | "github.com/hexops/gotextdiff/myers" 12 | "github.com/hexops/gotextdiff/span" 13 | "github.com/stretchr/testify/require" 14 | 15 | "github.com/sourcegraph/scip/bindings/go/scip" 16 | ) 17 | 18 | var updateSnapshots = flag.Bool("update-snapshots", false, "update SCIP snapshots files") 19 | 20 | type indexFunction = func(inputDirectory, outputDirectory string, sources []*scip.SourceFile) []*scip.SourceFile 21 | 22 | func SnapshotTest(t *testing.T, baseDir string, indexFunction indexFunction) { 23 | inputDirectory := filepath.Join(baseDir, "snapshots", "input") 24 | outputDirectory := filepath.Join(baseDir, "snapshots", "output") 25 | SnapshotTestDirectories(t, inputDirectory, outputDirectory, indexFunction) 26 | } 27 | 28 | func SnapshotTestDirectories(t *testing.T, inputDirectory, outputDirectory string, indexFunction indexFunction) { 29 | if *updateSnapshots { 30 | err := os.RemoveAll(outputDirectory) 31 | require.Nil(t, err) 32 | } 33 | testCases, err := os.ReadDir(inputDirectory) 34 | require.Nil(t, err) 35 | if len(testCases) == 0 { 36 | t.Fatalf("no subdirectories: %v", inputDirectory) 37 | } 38 | 39 | for _, testCase := range testCases { 40 | if !testCase.IsDir() { 41 | t.Fatalf("not a directory: %v", testCase.Name()) 42 | } 43 | t.Run(testCase.Name(), func(t *testing.T) { 44 | baseInputDirectory := filepath.Join(inputDirectory, testCase.Name()) 45 | baseOutputDirectory := filepath.Join(outputDirectory, testCase.Name()) 46 | sources, err := scip.NewSourcesFromDirectory(baseInputDirectory) 47 | require.Nil(t, err) 48 | obtainedSnapshots := indexFunction(baseInputDirectory, baseOutputDirectory, sources) 49 | snapshotTestSources(t, baseOutputDirectory, obtainedSnapshots) 50 | }) 51 | } 52 | } 53 | 54 | func snapshotTestSources(t *testing.T, outputDirectory string, obtainedSnapshots []*scip.SourceFile) { 55 | for _, document := range obtainedSnapshots { 56 | t.Run(document.RelativePath, func(t *testing.T) { 57 | obtained := document.Text 58 | outputFile := filepath.Join(outputDirectory, document.RelativePath) 59 | expected, err := os.ReadFile(outputFile) 60 | if err != nil { 61 | expected = []byte{} 62 | } 63 | if *updateSnapshots { 64 | err = os.MkdirAll(filepath.Dir(outputFile), 0755) 65 | require.Nil(t, err) 66 | err = os.WriteFile(outputFile, []byte(obtained), 0755) 67 | require.Nil(t, err) 68 | } else { 69 | edits := myers.ComputeEdits(span.URIFromPath(outputFile), string(expected), obtained) 70 | if len(edits) > 0 { 71 | diff := fmt.Sprint(gotextdiff.ToUnified( 72 | outputFile+" (obtained)", 73 | outputFile+" (expected)", 74 | string(expected), 75 | edits, 76 | )) 77 | t.Fatalf("\n" + diff) 78 | } 79 | } 80 | }) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /bindings/go/scip/tools.go: -------------------------------------------------------------------------------- 1 | //go:build tools 2 | // +build tools 3 | 4 | package scip 5 | 6 | // Hack for go.mod not supporting "dev dependencies" as a feature. 7 | // See https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module 8 | 9 | // Keep in sync with dev/proto-generate.sh 10 | import ( 11 | _ "github.com/bufbuild/buf/cmd/buf" 12 | _ "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc" 13 | _ "golang.org/x/tools/cmd/goimports" 14 | _ "google.golang.org/protobuf/cmd/protoc-gen-go" 15 | ) 16 | -------------------------------------------------------------------------------- /bindings/haskell/.gitignore: -------------------------------------------------------------------------------- 1 | dist-newstyle/ 2 | -------------------------------------------------------------------------------- /bindings/haskell/LICENSE: -------------------------------------------------------------------------------- 1 | ../../LICENSE -------------------------------------------------------------------------------- /bindings/haskell/README.md: -------------------------------------------------------------------------------- 1 | [![CI](https://github.com/sourcegraph/scip/actions/workflows/haskell.yml/badge.svg)](https://github.com/sourcegraph/scip/actions/workflows/haskell.yml/badge.svg) 2 | 3 | # Haskell bindings for SCIP 4 | 5 | These bindings use [Google's proto-lens generator for Haskell](https://github.com/google/proto-lens). 6 | 7 | Building: see the following workflow file for most up to date: 8 | https://github.com/sourcegraph/scip/blob/main/.github/workflows/haskell.yml 9 | 10 | First, get a working GHC environment: 11 | 12 | ```sh 13 | # Linux 14 | GHCUPVERSION=0.1.17.4 15 | curl --proto '=https' --tlsv1.2 -sSf https://downloads.haskell.org/~ghcup/$GHCUPVERSION/x86_64-linux-ghcup-$GHCUPVERSION > /usr/bin/ghcup && \ 16 | chmod +x /usr/bin/ghcup 17 | ghcup install ghc 8.10.7 --set 18 | ghcup install cabal 3.6.0.0 --set 19 | ``` 20 | 21 | ```sh 22 | # macOS 23 | GHCUPVERSION=0.1.17.4 24 | curl --proto '=https' --tlsv1.2 -sSf https://downloads.haskell.org/~ghcup/$GHCUPVERSION/x86_64-apple-darwin-ghcup-$GHCUPVERSION > /usr/local/bin/ghcup && \ 25 | chmod +x /usr/local/bin/ghcup 26 | ghcup install ghc 8.10.7 --set 27 | ghcup install cabal 3.6.0.0 --set 28 | ``` 29 | 30 | Next, install Google's `protoc` compiler. See: https://github.com/google/proto-lens/blob/master/docs/installing-protoc.md 31 | 32 | Then install the haskell generator from the proto-lens packages: 33 | 34 | ``` 35 | cabal install proto-lens-protoc 36 | ``` 37 | 38 | Finally, generate the source files manually in `src/*` 39 | 40 | ```sh 41 | # working directory: bindings/haskell 42 | protoc --plugin=protoc-gen-haskell=`which proto-lens-protoc` --haskell_out=src --proto_path=../.. scip.proto 43 | ``` 44 | 45 | Build the library as normal: 46 | 47 | ``` 48 | # working directory: bindings/haskell 49 | cabal build 50 | ``` 51 | -------------------------------------------------------------------------------- /bindings/haskell/scip.cabal: -------------------------------------------------------------------------------- 1 | cabal-version: 3.6 2 | 3 | name: scip 4 | version: 0.1.0.0 5 | synopsis: Haskell bindings for SCIP Code Intelligence Protocol 6 | license: Apache-2.0 7 | license-file: LICENSE 8 | author: SCIP Maintainers 9 | maintainer: code-intel@sourcegraph.com 10 | homepage: https://github.com/sourcegraph/scip 11 | category: Language 12 | build-type: Simple 13 | 14 | library 15 | ghc-options: -Wall 16 | exposed-modules: 17 | Proto.Scip, 18 | Proto.Scip_Fields 19 | hs-source-dirs: src 20 | default-language: Haskell2010 21 | build-depends: 22 | proto-lens-runtime ^>= 0.7.0.0, 23 | base ^>= 4.14.0.0 24 | -------------------------------------------------------------------------------- /bindings/rust/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "diff" 7 | version = "0.1.13" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" 10 | 11 | [[package]] 12 | name = "once_cell" 13 | version = "1.20.3" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" 16 | 17 | [[package]] 18 | name = "pretty_assertions" 19 | version = "1.4.1" 20 | source = "registry+https://github.com/rust-lang/crates.io-index" 21 | checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" 22 | dependencies = [ 23 | "diff", 24 | "yansi", 25 | ] 26 | 27 | [[package]] 28 | name = "proc-macro2" 29 | version = "1.0.94" 30 | source = "registry+https://github.com/rust-lang/crates.io-index" 31 | checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" 32 | dependencies = [ 33 | "unicode-ident", 34 | ] 35 | 36 | [[package]] 37 | name = "protobuf" 38 | version = "3.7.2" 39 | source = "registry+https://github.com/rust-lang/crates.io-index" 40 | checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" 41 | dependencies = [ 42 | "once_cell", 43 | "protobuf-support", 44 | "thiserror", 45 | ] 46 | 47 | [[package]] 48 | name = "protobuf-support" 49 | version = "3.7.2" 50 | source = "registry+https://github.com/rust-lang/crates.io-index" 51 | checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" 52 | dependencies = [ 53 | "thiserror", 54 | ] 55 | 56 | [[package]] 57 | name = "quote" 58 | version = "1.0.39" 59 | source = "registry+https://github.com/rust-lang/crates.io-index" 60 | checksum = "c1f1914ce909e1658d9907913b4b91947430c7d9be598b15a1912935b8c04801" 61 | dependencies = [ 62 | "proc-macro2", 63 | ] 64 | 65 | [[package]] 66 | name = "scip" 67 | version = "0.5.2" 68 | dependencies = [ 69 | "pretty_assertions", 70 | "protobuf", 71 | ] 72 | 73 | [[package]] 74 | name = "syn" 75 | version = "2.0.100" 76 | source = "registry+https://github.com/rust-lang/crates.io-index" 77 | checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" 78 | dependencies = [ 79 | "proc-macro2", 80 | "quote", 81 | "unicode-ident", 82 | ] 83 | 84 | [[package]] 85 | name = "thiserror" 86 | version = "1.0.69" 87 | source = "registry+https://github.com/rust-lang/crates.io-index" 88 | checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" 89 | dependencies = [ 90 | "thiserror-impl", 91 | ] 92 | 93 | [[package]] 94 | name = "thiserror-impl" 95 | version = "1.0.69" 96 | source = "registry+https://github.com/rust-lang/crates.io-index" 97 | checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" 98 | dependencies = [ 99 | "proc-macro2", 100 | "quote", 101 | "syn", 102 | ] 103 | 104 | [[package]] 105 | name = "unicode-ident" 106 | version = "1.0.18" 107 | source = "registry+https://github.com/rust-lang/crates.io-index" 108 | checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" 109 | 110 | [[package]] 111 | name = "yansi" 112 | version = "1.0.1" 113 | source = "registry+https://github.com/rust-lang/crates.io-index" 114 | checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" 115 | -------------------------------------------------------------------------------- /bindings/rust/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scip" 3 | version = "0.5.2" 4 | edition = "2021" 5 | license = "Apache-2.0" 6 | description = """ 7 | SCIP (pronunciation: "skip") is a language-agnostic protocol for indexing source code, which can be used to power code navigation functionality such as Go to definition, Find references, and Find implementations. 8 | """ 9 | repository = "https://github.com/sourcegraph/scip" 10 | readme = "../../README.md" 11 | 12 | # We need >= 1.81.0 because: we only test against that 13 | # in the CI (we use protobuf-codegen in CI, and that 14 | # has home@0.5.11 as a dependency, which requires Rust >= 1.81.0). 15 | rust-version = "1.81.0" 16 | 17 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 18 | 19 | [dependencies] 20 | # Keep in sync with dev/proto-generate.sh 21 | protobuf = "=3.7.2" 22 | 23 | [dev-dependencies] 24 | pretty_assertions = "1.2.1" 25 | 26 | [lib] 27 | name = "scip" 28 | path = "src/mod.rs" 29 | # disable doctests because generated/scip.rs has a perceived doctest. 30 | doctest = false 31 | -------------------------------------------------------------------------------- /bindings/rust/LICENSE: -------------------------------------------------------------------------------- 1 | ../../LICENSE -------------------------------------------------------------------------------- /bindings/rust/src/generated/mod.rs: -------------------------------------------------------------------------------- 1 | // @generated 2 | 3 | pub mod scip; 4 | -------------------------------------------------------------------------------- /bindings/rust/src/mod.rs: -------------------------------------------------------------------------------- 1 | // SCIP bindings for rust 2 | 3 | // This will pull generated code into `scip::types` 4 | #[path = "generated/mod.rs"] 5 | mod scip_mod; 6 | pub use scip_mod::scip as types; 7 | 8 | // Exports symbol usage under scip::symbol namespace 9 | pub mod symbol; 10 | 11 | /// Write a message to a particular filepath. 12 | /// 13 | /// This allows users of the SCIP library to not add protobuf as 14 | /// a direct dependency of the project (which can be useful to limit 15 | /// usage of protobuf elsewhere if not desired). 16 | pub fn write_message_to_file

( 17 | path: P, 18 | msg: impl protobuf::Message, 19 | ) -> Result<(), Box> 20 | where 21 | P: AsRef, 22 | { 23 | use std::io::Write; 24 | 25 | let res = msg.write_to_bytes()?; 26 | let output = std::fs::File::create(path)?; 27 | let mut writer = std::io::BufWriter::new(output); 28 | writer.write_all(&res)?; 29 | 30 | Ok(()) 31 | } 32 | -------------------------------------------------------------------------------- /bindings/typescript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@sourcegraph/scip", 3 | "version": "0.1.0", 4 | "description": "Generated TypeScript definitions for the Semantic Code Intelligence Protocol.", 5 | "repository": "https://github.com/sourcegraph/scip/bindings/typescript", 6 | "author": "Sourcegraph", 7 | "license": "Apache-2.0", 8 | "sideEffects": false, 9 | "private": false, 10 | "dependencies": { 11 | "google-protobuf": "^3.20.1" 12 | }, 13 | "devDependencies": { 14 | "@types/google-protobuf": "3.15.6", 15 | "protoc-gen-ts": "0.8.6", 16 | "typescript": "^4.9.0" 17 | }, 18 | "comment": [ 19 | "protoc-gen-ts is pinned to 0.8.1 because there is a breaking change in ", 20 | "0.8.2 which causes a crash when generating output: ", 21 | " ", 22 | "const { major = 0, minor = 0, patch = 0 } = request.compiler_version; ", 23 | " ^ ", 24 | "TypeError: Cannot read property 'major' of undefined ", 25 | " at Object. (/Users/varun/Code/scip/node_modules/protoc-gen-ts/src/index.js:60:13)", 26 | " at Module._compile (node:internal/modules/cjs/loader:1101:14) ", 27 | " ", 28 | "We ran into this with https://github.com/sourcegraph/scip/pull/1 " 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /bindings/typescript/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2016", 4 | "module": "commonjs", 5 | "rootDir": ".", 6 | "strict": true 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /buf.gen.yaml: -------------------------------------------------------------------------------- 1 | # Configuration file for https://buf.build/, which we use for Protobuf code generation. 2 | version: v1 3 | plugins: 4 | - name: doc 5 | path: ./.bin/protoc-gen-doc 6 | out: ./docs/ 7 | opt: docs/scip.sprig,scip.md 8 | # NOTE: the sprig file is a template, it is not generated. 9 | - name: go 10 | out: ./bindings/go/scip/ 11 | opt: 12 | - paths=source_relative 13 | - name: ts 14 | path: ./bindings/typescript/node_modules/.bin/protoc-gen-ts 15 | out: ./bindings/typescript/ 16 | - name: rust 17 | path: ./.bin/bin/protoc-gen-rs 18 | out: ./bindings/rust/src/generated/ 19 | - name: haskell 20 | out: ./bindings/haskell/src 21 | -------------------------------------------------------------------------------- /buf.work.yaml: -------------------------------------------------------------------------------- 1 | # Configuration file for https://buf.build/, which we use for Protobuf code generation. 2 | version: v1 3 | directories: 4 | - . 5 | -------------------------------------------------------------------------------- /buf.yaml: -------------------------------------------------------------------------------- 1 | # Configuration file for https://buf.build/, which we use for Protobuf code generation. 2 | version: v1 3 | lint: 4 | use: 5 | - DEFAULT 6 | except: 7 | - PACKAGE_VERSION_SUFFIX 8 | - ENUM_VALUE_PREFIX 9 | - ENUM_VALUE_UPPER_SNAKE_CASE 10 | - ENUM_ZERO_VALUE_SUFFIX 11 | breaking: 12 | use: 13 | - FILE 14 | -------------------------------------------------------------------------------- /cmd/scip/convert_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "cmp" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/klauspost/compress/zstd" 10 | "github.com/stretchr/testify/require" 11 | "golang.org/x/exp/slices" 12 | "zombiezen.com/go/sqlite" 13 | "zombiezen.com/go/sqlite/sqlitex" 14 | 15 | "github.com/sourcegraph/scip/bindings/go/scip" 16 | ) 17 | 18 | func TestConvert_SmokeTest(t *testing.T) { 19 | // Create temp directory for test DB 20 | tempDir := t.TempDir() 21 | 22 | sqliteDBPath := filepath.Join(tempDir, "index.db") 23 | 24 | index := testIndex1() 25 | 26 | db, err := createSQLiteDatabase(sqliteDBPath) 27 | require.NoError(t, err) 28 | defer func() { require.NoError(t, db.Close()) }() 29 | 30 | writer, err := zstd.NewWriter(nil) 31 | require.NoError(t, err) 32 | converter := NewConverter(db, chunkSizeHint, writer) 33 | err = converter.Convert(index) 34 | require.NoError(t, err) 35 | 36 | checks := []struct { 37 | name string 38 | fn func(*testing.T, *scip.Index, *sqlite.Conn) 39 | }{ 40 | {"documents", checkDocuments}, 41 | {"symbols", checkSymbols}, 42 | {"occurrences", checkOccurrences}, 43 | } 44 | 45 | for _, check := range checks { 46 | t.Run(check.name, func(t *testing.T) { 47 | check.fn(t, index, db) 48 | }) 49 | } 50 | } 51 | 52 | func testIndex1() *scip.Index { 53 | pkg1S1Sym := "scip-go go . . pkg1/S1#" 54 | return &scip.Index{ 55 | Documents: []*scip.Document{ 56 | { 57 | RelativePath: "a.go", 58 | Occurrences: []*scip.Occurrence{ 59 | {Symbol: pkg1S1Sym, Range: []int32{10, 3, 6}, SymbolRoles: int32(scip.SymbolRole_Definition)}, 60 | }, 61 | Symbols: []*scip.SymbolInformation{ 62 | {Symbol: pkg1S1Sym}, 63 | }, 64 | }, 65 | { 66 | RelativePath: "b.go", 67 | Occurrences: []*scip.Occurrence{ 68 | {Symbol: pkg1S1Sym, Range: []int32{15, 9, 12}}, 69 | }, 70 | }, 71 | }, 72 | } 73 | } 74 | 75 | func checkDocuments(t *testing.T, index *scip.Index, db *sqlite.Conn) { 76 | query := "SELECT relative_path FROM documents" 77 | var dbPaths []string 78 | err := sqlitex.ExecuteTransient(db, query, &sqlitex.ExecOptions{ 79 | ResultFunc: func(stmt *sqlite.Stmt) error { 80 | dbPaths = append(dbPaths, stmt.ColumnText(0)) 81 | return nil 82 | }, 83 | }) 84 | require.NoError(t, err) 85 | var expectedPaths []string 86 | for _, doc := range index.Documents { 87 | expectedPaths = append(expectedPaths, doc.RelativePath) 88 | } 89 | slices.Sort(expectedPaths) 90 | expectedPaths = slices.Compact(expectedPaths) 91 | slices.Sort(dbPaths) 92 | 93 | require.Equal(t, expectedPaths, dbPaths) 94 | } 95 | 96 | func checkSymbols(t *testing.T, index *scip.Index, db *sqlite.Conn) { 97 | query := "SELECT symbol FROM global_symbols" 98 | var dbSymbols []string 99 | err := sqlitex.ExecuteTransient(db, query, &sqlitex.ExecOptions{ 100 | ResultFunc: func(stmt *sqlite.Stmt) error { 101 | dbSymbols = append(dbSymbols, stmt.ColumnText(0)) 102 | return nil 103 | }, 104 | }) 105 | require.NoError(t, err) 106 | 107 | var expectedSymbols []string 108 | for _, doc := range index.Documents { 109 | for _, occ := range doc.Occurrences { 110 | expectedSymbols = append(expectedSymbols, occ.Symbol) 111 | } 112 | for _, sym := range doc.Symbols { 113 | expectedSymbols = append(expectedSymbols, sym.Symbol) 114 | } 115 | } 116 | slices.Sort(expectedSymbols) 117 | expectedSymbols = slices.Compact(expectedSymbols) 118 | slices.Sort(dbSymbols) 119 | 120 | require.Equal(t, expectedSymbols, dbSymbols) 121 | } 122 | 123 | func checkOccurrences(t *testing.T, index *scip.Index, db *sqlite.Conn) { 124 | zstdReader, err := zstd.NewReader(bytes.NewBuffer(nil)) 125 | require.NoError(t, err) 126 | 127 | query := `SELECT d.relative_path, occurrences 128 | FROM documents d 129 | JOIN chunks c ON c.document_id = d.id` 130 | dbOccurrences := []occurrenceData{} 131 | err = sqlitex.ExecuteTransient(db, query, &sqlitex.ExecOptions{ 132 | ResultFunc: func(stmt *sqlite.Stmt) error { 133 | var c Chunk 134 | err = c.fromDBFormat(stmt.ColumnReader(1), zstdReader) 135 | require.NoError(t, err) 136 | for _, occ := range c.Occurrences { 137 | dbOccurrences = append(dbOccurrences, occurrenceData{ 138 | DocumentPath: stmt.ColumnText(0), 139 | Symbol: occ.Symbol, 140 | Role: occ.SymbolRoles, 141 | Range: scip.NewRangeUnchecked(occ.Range), 142 | }) 143 | } 144 | return nil 145 | }, 146 | }) 147 | require.NoError(t, err) 148 | cmpFn := func(a, b occurrenceData) int { 149 | return cmp.Or(cmp.Compare(a.DocumentPath, b.DocumentPath), 150 | a.Range.CompareStrict(b.Range)) 151 | } 152 | slices.SortFunc(dbOccurrences, cmpFn) 153 | 154 | var expectedOccurrences []occurrenceData 155 | for _, doc := range index.Documents { 156 | for _, occ := range doc.Occurrences { 157 | expectedOccurrences = append(expectedOccurrences, occurrenceData{ 158 | DocumentPath: doc.RelativePath, 159 | Symbol: occ.Symbol, 160 | Role: occ.SymbolRoles, 161 | Range: scip.NewRangeUnchecked(occ.Range), 162 | }) 163 | } 164 | } 165 | slices.SortFunc(expectedOccurrences, cmpFn) 166 | 167 | require.Equal(t, expectedOccurrences, dbOccurrences) 168 | } 169 | 170 | type occurrenceData struct { 171 | DocumentPath string 172 | Symbol string 173 | Role int32 174 | Range scip.Range 175 | } 176 | -------------------------------------------------------------------------------- /cmd/scip/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | _ "embed" 5 | "fmt" 6 | "log" 7 | "os" 8 | "runtime/debug" 9 | "strings" 10 | 11 | "github.com/urfave/cli/v2" 12 | ) 13 | 14 | func main() { 15 | app := scipApp() 16 | if err := app.Run(os.Args); err != nil { 17 | log.Fatal(err) 18 | } 19 | } 20 | 21 | func commands() []*cli.Command { 22 | lint := lintCommand() 23 | print := printCommand() 24 | snapshot := snapshotCommand() 25 | stats := statsCommand() 26 | test := testCommand() 27 | convert := convertCommand() 28 | return []*cli.Command{&lint, &print, &snapshot, &stats, &test, &convert} 29 | } 30 | 31 | //go:embed version.txt 32 | var version string 33 | 34 | var Reproducible = "" // set by ldflags in CI 35 | 36 | func gitSuffix() string { 37 | if Reproducible != "" { 38 | return "" 39 | } 40 | var rev, timestamp string 41 | clean := "true" 42 | if buildInfo, ok := debug.ReadBuildInfo(); ok { 43 | for _, kv := range buildInfo.Settings { 44 | switch kv.Key { 45 | case "vcs.revision": 46 | rev = kv.Value 47 | case "vcs.time": 48 | timestamp = kv.Value 49 | case "vcs.modified": 50 | if kv.Value == "true" { 51 | clean = "false" 52 | } 53 | } 54 | } 55 | } 56 | return fmt.Sprintf("-dev\nSHA: %s\ntimestamp: %s\nclean: %s", rev, timestamp, clean) 57 | } 58 | 59 | func scipApp() *cli.App { 60 | app := &cli.App{ 61 | Name: "scip", 62 | Version: fmt.Sprintf("v%s%s", strings.TrimSpace(version), gitSuffix()), 63 | Usage: "SCIP Code Intelligence Protocol CLI", 64 | Description: "For more details, see the project README at:\n\n\thttps://github.com/sourcegraph/scip", 65 | Commands: commands(), 66 | } 67 | return app 68 | } 69 | 70 | func fromFlag(storage *string) *cli.StringFlag { 71 | return &cli.StringFlag{ 72 | Name: "from", 73 | Usage: "Path to SCIP index file", 74 | Destination: storage, 75 | Value: "index.scip", 76 | } 77 | } 78 | 79 | func commentSyntaxFlag(storage *string) *cli.StringFlag { 80 | return &cli.StringFlag{ 81 | Name: "comment-syntax", 82 | Usage: "Comment syntax to use for snapshot files", 83 | Destination: storage, 84 | Value: "//", 85 | } 86 | } 87 | 88 | func projectRootFlag(storage *string) *cli.StringFlag { 89 | return &cli.StringFlag{ 90 | Name: "project-root", 91 | Usage: "Override project root in the SCIP file. " + 92 | "This can be helpful when the SCIP index was created on another computer", 93 | Destination: storage, 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /cmd/scip/option_from.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "strings" 7 | 8 | "github.com/cockroachdb/errors" 9 | "github.com/sourcegraph/scip/bindings/go/scip" 10 | "google.golang.org/protobuf/proto" 11 | ) 12 | 13 | // readFromOptions reads the fromPath parameter into a SCIP Index message. 14 | // If fromPath has the value "-" then the SCIP Index is read from os.Stdin. 15 | // Otherwise, fromPath is interpreted as a file path and the bytes are read from disk. 16 | func readFromOption(fromPath string) (*scip.Index, error) { 17 | var scipReader io.Reader 18 | if fromPath == "-" { 19 | scipReader = os.Stdin 20 | } else if !strings.HasSuffix(fromPath, ".scip") && !strings.HasSuffix(fromPath, ".lsif-typed") { 21 | return nil, errors.Newf("expected file with .scip extension but found %s", fromPath) 22 | } else { 23 | scipFile, err := os.Open(fromPath) 24 | defer scipFile.Close() 25 | if err != nil { 26 | return nil, err 27 | } 28 | scipReader = scipFile 29 | } 30 | 31 | scipBytes, err := io.ReadAll(scipReader) 32 | if err != nil { 33 | return nil, errors.Wrapf(err, "failed to read SCIP index at path %s", fromPath) 34 | } 35 | 36 | scipIndex := scip.Index{} 37 | err = proto.Unmarshal(scipBytes, &scipIndex) 38 | if err != nil { 39 | return nil, errors.Wrapf(err, "failed to parse SCIP index at path %s", fromPath) 40 | } 41 | return &scipIndex, nil 42 | } 43 | -------------------------------------------------------------------------------- /cmd/scip/print.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "strings" 7 | 8 | "github.com/bytedance/sonic" 9 | "github.com/cockroachdb/errors" 10 | "github.com/k0kubun/pp/v3" 11 | "github.com/urfave/cli/v2" 12 | ) 13 | 14 | func printCommand() cli.Command { 15 | var json, colorOutput bool 16 | snapshot := cli.Command{ 17 | Name: "print", 18 | Usage: "Print a SCIP index for debugging", 19 | Description: `WARNING: The TTY output may change over time. 20 | Do not rely on non-JSON output in scripts`, 21 | Flags: []cli.Flag{ 22 | &cli.BoolFlag{ 23 | Name: "json", 24 | Usage: "Output in JSON format", 25 | Destination: &json, 26 | }, 27 | &cli.BoolFlag{ 28 | Name: "color", 29 | Usage: "Enable color output for TTY (no effect for JSON)", 30 | Destination: &colorOutput, 31 | Value: true, 32 | DefaultText: "true", 33 | }, 34 | }, 35 | Action: func(c *cli.Context) error { 36 | indexPath := c.Args().Get(0) 37 | if indexPath == "" { 38 | return errors.New("missing argument for path to SCIP index") 39 | } 40 | // Following https://no-color.org/ 41 | if val, found := os.LookupEnv("NO_COLOR"); found && val != "" { 42 | switch strings.ToLower(val) { 43 | case "": 44 | break 45 | case "0", "false", "off": 46 | colorOutput = false 47 | default: 48 | colorOutput = true 49 | } 50 | } 51 | return printMain(indexPath, colorOutput, json, c.App.Writer) 52 | }, 53 | } 54 | return snapshot 55 | } 56 | 57 | func printMain(indexPath string, colorOutput bool, json bool, out io.Writer) error { 58 | index, err := readFromOption(indexPath) 59 | if err != nil { 60 | return err 61 | } 62 | if json { 63 | encoder := sonic.ConfigDefault.NewEncoder(out) 64 | return encoder.Encode(index) 65 | } else { 66 | prettyPrinter := pp.New() 67 | prettyPrinter.SetColoringEnabled(colorOutput) 68 | prettyPrinter.SetExportedOnly(true) 69 | prettyPrinter.SetOutput(out) 70 | _, err = prettyPrinter.Print(index) 71 | return err 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /cmd/scip/print_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "testing" 11 | 12 | "github.com/sourcegraph/scip/bindings/go/scip" 13 | "github.com/stretchr/testify/require" 14 | "google.golang.org/protobuf/proto" 15 | ) 16 | 17 | func TestJSONPrinting(t *testing.T) { 18 | app := scipApp() 19 | // Redirect CLI writer to a buffer 20 | var buff bytes.Buffer 21 | app.Writer = io.Writer(&buff) 22 | idx := makeIndex([]string{"a"}, stringMap{"f": {"b"}}, stringMap{"f": {"a", "b"}}) 23 | 24 | idx.Metadata = &scip.Metadata{ProjectRoot: "howdy"} 25 | 26 | // Serialise SCIP index 27 | indexBytes, err := proto.Marshal(idx) 28 | 29 | // Write SCIP index to a temp file 30 | dir := os.TempDir() 31 | file, err := ioutil.TempFile(dir, "scip-cli-json-test*.scip") 32 | if err != nil { 33 | log.Fatal(err) 34 | } 35 | defer os.Remove(file.Name()) 36 | 37 | _, fErr := file.Write(indexBytes) 38 | if fErr != nil { 39 | log.Fatal(fErr) 40 | } 41 | 42 | // Run the JSON command with the temporary file 43 | runErr := app.Run([]string{"scip", "print", "--json", file.Name()}) 44 | if runErr != nil { 45 | log.Fatal(runErr) 46 | } 47 | 48 | type JsonIndex struct { 49 | Metadata struct { 50 | ProjectRoot string `json:"project_root"` 51 | } 52 | Documents []struct { 53 | RelativePath string `json:"relative_path"` 54 | } `json:"documents"` 55 | } 56 | 57 | var roundtripResult JsonIndex 58 | 59 | bytes := buff.Bytes() 60 | 61 | jsonErr := json.Unmarshal(bytes, &roundtripResult) 62 | if jsonErr != nil { 63 | log.Fatal(jsonErr) 64 | } 65 | 66 | require.Equal(t, "howdy", roundtripResult.Metadata.ProjectRoot) 67 | require.Equal(t, 1, len(roundtripResult.Documents)) 68 | require.Equal(t, "f", roundtripResult.Documents[0].RelativePath) 69 | } 70 | -------------------------------------------------------------------------------- /cmd/scip/snapshot.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/urfave/cli/v2" 9 | 10 | "github.com/cockroachdb/errors" 11 | "github.com/sourcegraph/scip/bindings/go/scip" 12 | "github.com/sourcegraph/scip/bindings/go/scip/testutil" 13 | ) 14 | 15 | type snapshotFlags struct { 16 | from string 17 | output string 18 | customProjectRoot string 19 | strict bool 20 | commentSyntax string 21 | } 22 | 23 | func snapshotCommand() cli.Command { 24 | var snapshotFlags snapshotFlags 25 | snapshot := cli.Command{ 26 | Name: "snapshot", 27 | Usage: "Generate snapshot files for golden testing", 28 | Description: `The snapshot subcommand generates snapshot files which 29 | can be use for inspecting the output of an index in a 30 | visual way. Occurrences are marked with caret signs (^) 31 | and symbol information. 32 | 33 | For testing a SCIP indexer, you can either use this subcommand 34 | along with 'git diff' or equivalent, or you can use the dedicated 35 | 'test' subcommand for more targeted checks. 36 | `, 37 | Flags: []cli.Flag{ 38 | fromFlag(&snapshotFlags.from), 39 | &cli.StringFlag{ 40 | Name: "to", 41 | Usage: "Path to output directory for snapshot files", 42 | Destination: &snapshotFlags.output, 43 | Value: "scip-snapshot", 44 | }, 45 | projectRootFlag(&snapshotFlags.customProjectRoot), 46 | &cli.BoolFlag{ 47 | Name: "strict", 48 | Usage: "If true, fail fast on errors", 49 | Destination: &snapshotFlags.strict, 50 | Value: true, 51 | }, 52 | commentSyntaxFlag(&snapshotFlags.commentSyntax), 53 | }, 54 | Action: func(c *cli.Context) error { 55 | return snapshotMain(snapshotFlags) 56 | }, 57 | } 58 | return snapshot 59 | } 60 | 61 | func snapshotMain(flags snapshotFlags) error { 62 | from := flags.from 63 | index, err := readFromOption(from) 64 | if err != nil { 65 | return err 66 | } 67 | output := flags.output 68 | err = os.RemoveAll(output) 69 | if err != nil { 70 | return err 71 | } 72 | symbolFormatter := scip.LenientVerboseSymbolFormatter 73 | if flags.strict { 74 | symbolFormatter = scip.VerboseSymbolFormatter 75 | symbolFormatter.OnError = func(err error) error { 76 | return errors.Wrap(err, "use --strict=false to ignore this error") 77 | } 78 | } 79 | snapshots, err := testutil.FormatSnapshots(index, flags.commentSyntax, symbolFormatter, flags.customProjectRoot) 80 | if err != nil { 81 | return err 82 | } 83 | for _, snapshot := range snapshots { 84 | outputPath := filepath.Join(output, snapshot.RelativePath) 85 | err = os.MkdirAll(filepath.Dir(outputPath), 0755) 86 | if err != nil { 87 | return err 88 | } 89 | err = os.WriteFile(outputPath, []byte(snapshot.Text), 0755) 90 | if err != nil { 91 | return err 92 | } 93 | } 94 | fmt.Println("done: " + output) 95 | return nil 96 | } 97 | -------------------------------------------------------------------------------- /cmd/scip/stats.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "net/url" 8 | "os" 9 | "path/filepath" 10 | 11 | "github.com/hhatto/gocloc" 12 | "github.com/montanaflynn/stats" 13 | "github.com/urfave/cli/v2" 14 | "google.golang.org/protobuf/proto" 15 | 16 | "github.com/cockroachdb/errors" 17 | 18 | "github.com/sourcegraph/scip/bindings/go/scip" 19 | ) 20 | 21 | type statsFlags struct { 22 | from string 23 | customProjectRoot string 24 | } 25 | 26 | func statsCommand() cli.Command { 27 | var statsFlags statsFlags 28 | stats := cli.Command{ 29 | Name: "stats", 30 | Usage: "Output useful statistics about a SCIP index", 31 | Flags: []cli.Flag{ 32 | fromFlag(&statsFlags.from), 33 | projectRootFlag(&statsFlags.customProjectRoot), 34 | }, 35 | Action: func(c *cli.Context) error { 36 | return statsMain(statsFlags) 37 | }, 38 | } 39 | return stats 40 | } 41 | 42 | func statsMain(flags statsFlags) error { 43 | from := flags.from 44 | index, err := readFromOption(from) 45 | if err != nil { 46 | return errors.Wrap(err, "error reading SCIP file") 47 | } 48 | if index.Metadata == nil { 49 | return errors.Errorf("Index.Metadata is nil (--from=%s)", from) 50 | } 51 | output := map[string]interface{}{} 52 | indexStats, err := countStatistics(index, flags.customProjectRoot) 53 | if err != nil { 54 | return errors.Wrap(err, "error counting stats") 55 | } 56 | jsonBytes, err := json.MarshalIndent(indexStats, "", " ") 57 | if err != nil { 58 | return errors.Wrapf(err, "failed to marshall into JSON %s", output) 59 | } 60 | fmt.Println(string(jsonBytes)) 61 | return nil 62 | } 63 | 64 | type Stats struct { 65 | Percentiles struct { 66 | Fifty int32 `json:"50"` 67 | Ninety int32 `json:"90"` 68 | NinetyFive int32 `json:"95"` 69 | NinetyNine int32 `json:"99"` 70 | NinetyNinePointNine int32 `json:"99.9"` 71 | } 72 | Mean int32 `json:"mean"` 73 | Stddev int32 `json:"stddev"` 74 | Max int32 `json:"max"` 75 | Sum int32 `json:"sum"` 76 | Comment string `json:"comment"` 77 | } 78 | 79 | func NewStats(values []float64) Stats { 80 | s := Stats{} 81 | s.Percentiles.Fifty = percentile(values, 50) 82 | s.Percentiles.Ninety = percentile(values, 90) 83 | s.Percentiles.NinetyFive = percentile(values, 95) 84 | s.Percentiles.NinetyNine = percentile(values, 99) 85 | s.Percentiles.NinetyNinePointNine = percentile(values, 99.9) 86 | mean, _ := stats.Mean(values) 87 | s.Mean = int32(mean) 88 | stddev, _ := stats.StandardDeviation(values) 89 | s.Stddev = int32(stddev) 90 | max, _ := stats.Max(values) 91 | s.Max = int32(max) 92 | sum, _ := stats.Sum(values) 93 | s.Sum = int32(sum) 94 | return s 95 | } 96 | 97 | type indexStatistics struct { 98 | Documents int32 `json:"documents"` 99 | DocumentSizes Stats `json:"documentSizes"` 100 | LinesOfCode int32 `json:"linesOfCode"` 101 | Occurrences int32 `json:"occurrences"` 102 | OccurrenceCounts Stats `json:"occurrenceCounts"` 103 | Definitions int32 `json:"definitions"` 104 | DefinitionCounts Stats `json:"definitionCounts"` 105 | } 106 | 107 | func countStatistics(index *scip.Index, customProjectRoot string) (*indexStatistics, error) { 108 | loc, err := countLinesOfCode(index, customProjectRoot) 109 | var linesOfCode int32 110 | if err != nil { 111 | // Keep this a non-fatal error so that we can measure other index stats 112 | // even if the project is not cloned locally (e.g. if it's a huge 113 | // project like Chromium or the Linux kernel). 114 | log.Printf("Couldn't count lines of code: %s", err) 115 | } else { 116 | linesOfCode = loc.Total.Code 117 | } 118 | stats := &indexStatistics{ 119 | Documents: int32(len(index.Documents)), 120 | LinesOfCode: linesOfCode, 121 | Occurrences: 0, 122 | Definitions: 0, 123 | } 124 | documentSizes := []float64{} 125 | occurrenceCounts := []float64{} 126 | definitionCounts := []float64{} 127 | for _, document := range index.Documents { 128 | bytes, _ := proto.Marshal(document) 129 | documentSizes = append(documentSizes, float64(len(bytes))) 130 | stats.Occurrences += int32(len(document.Occurrences)) 131 | occurrenceCounts = append(occurrenceCounts, float64(len(document.Occurrences))) 132 | definitionCounts = append(definitionCounts, 0) 133 | for _, occurrence := range document.Occurrences { 134 | if scip.SymbolRole_Definition.Matches(occurrence) { 135 | stats.Definitions += 1 136 | definitionCounts[len(definitionCounts)-1] += 1 137 | } 138 | } 139 | } 140 | stats.DocumentSizes = NewStats(documentSizes) 141 | stats.DocumentSizes.Comment = "sizes are in bytes" 142 | stats.OccurrenceCounts = NewStats(occurrenceCounts) 143 | stats.DefinitionCounts = NewStats(definitionCounts) 144 | stats.DefinitionCounts.Comment = "counted using occurrences" 145 | return stats, nil 146 | } 147 | 148 | func percentile(buf []float64, percent float64) int32 { 149 | res, _ := stats.Percentile(buf, percent) 150 | return int32(res) 151 | } 152 | 153 | func countLinesOfCode(index *scip.Index, customProjectRoot string) (*gocloc.Result, error) { 154 | var localSource string 155 | root, err := url.Parse(index.Metadata.ProjectRoot) 156 | if err != nil { 157 | return nil, errors.Wrapf(err, "failed to parse Index.Metadata.ProjectRoot as a URI %s", index.Metadata.ProjectRoot) 158 | } 159 | if customProjectRoot != "" { 160 | localSource = customProjectRoot 161 | } else { 162 | _, err := os.Stat(root.Path) 163 | if errors.Is(err, os.ErrNotExist) { 164 | cwd, _ := os.Getwd() 165 | log.Printf("Project root [%s] doesn't exist, using current working directory [%s] instead. "+ 166 | "To override this behaviour, specify --project-root= option", root.Path, cwd) 167 | localSource = cwd 168 | } else if err != nil { 169 | return nil, err 170 | } 171 | } 172 | 173 | stat, err := os.Stat(localSource) 174 | if err != nil { 175 | return nil, err 176 | } 177 | if !stat.IsDir() { 178 | return nil, errors.Errorf("Project root [%s] is not a directory", localSource) 179 | } 180 | processor := gocloc.NewProcessor(gocloc.NewDefinedLanguages(), gocloc.NewClocOptions()) 181 | var paths []string 182 | for _, document := range index.Documents { 183 | paths = append(paths, filepath.Join(localSource, document.RelativePath)) 184 | } 185 | return processor.Analyze(paths) 186 | } 187 | -------------------------------------------------------------------------------- /cmd/scip/test_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "golang.org/x/exp/slices" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestTestCasesForLine(t *testing.T) { 12 | actual, linesUsed := testCasesForLine( 13 | 2, 14 | []string{ 15 | "void main() {", 16 | " // ^ definition ", 17 | " print(foo)", 18 | " // ^^^ reference ", 19 | " // ^ reference ", 20 | " // <- reference ", 21 | " // ^ diagnostic Warning", 22 | " // > with multiline", 23 | " // > additional data", 24 | " final test = 4", 25 | " ^ definition ", 26 | "}", 27 | }, 28 | "//", 29 | ) 30 | 31 | // total of 4 test cases on the specified line 32 | require.Equal(t, 4, len(actual)) 33 | require.Equal(t, 6, linesUsed) 34 | 35 | // only the first test case has enforceLength = true 36 | require.True(t, actual[0].enforceLength) 37 | require.Equal(t, 3, actual[0].attribute.length) 38 | require.False(t, actual[1].enforceLength) 39 | require.False(t, actual[2].enforceLength) 40 | require.False(t, actual[3].enforceLength) 41 | 42 | // all test cases should have the same start 43 | require.Equal(t, 8, actual[0].attribute.start) 44 | require.Equal(t, 9, actual[1].attribute.start) // different start, same symbol 45 | require.Equal(t, 8, actual[2].attribute.start) 46 | require.Equal(t, 8, actual[3].attribute.start) 47 | 48 | // validate kind on each test case 49 | require.Equal(t, referenceAttrKind, actual[0].attribute.kind) 50 | require.Equal(t, referenceAttrKind, actual[1].attribute.kind) 51 | require.Equal(t, referenceAttrKind, actual[2].attribute.kind) 52 | require.Equal(t, diagnosticAttrKind, actual[3].attribute.kind) 53 | 54 | // validate that additionalData is correctly parsed 55 | require.True( 56 | t, 57 | slices.Equal( 58 | actual[3].attribute.additionalData, 59 | []string{"with multiline", "additional data"}, 60 | ), 61 | ) 62 | } 63 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/.gitattributes: -------------------------------------------------------------------------------- 1 | bindings/node/** linguist-generated=true 2 | bindings/rust/** linguist-generated=true 3 | src/grammar.json linguist-generated=true 4 | src/node-types.json linguist-generated=true 5 | src/parser.c linguist-generated=true 6 | src/tree_sitter/parser.h linguist-generated=true 7 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "0.7.18" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "cc" 16 | version = "1.0.73" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" 19 | 20 | [[package]] 21 | name = "memchr" 22 | version = "2.5.0" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" 25 | 26 | [[package]] 27 | name = "regex" 28 | version = "1.5.5" 29 | source = "registry+https://github.com/rust-lang/crates.io-index" 30 | checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" 31 | dependencies = [ 32 | "aho-corasick", 33 | "memchr", 34 | "regex-syntax", 35 | ] 36 | 37 | [[package]] 38 | name = "regex-syntax" 39 | version = "0.6.25" 40 | source = "registry+https://github.com/rust-lang/crates.io-index" 41 | checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" 42 | 43 | [[package]] 44 | name = "tree-sitter" 45 | version = "0.20.6" 46 | source = "registry+https://github.com/rust-lang/crates.io-index" 47 | checksum = "09b3b781640108d29892e8b9684642d2cda5ea05951fd58f0fea1db9edeb9b71" 48 | dependencies = [ 49 | "cc", 50 | "regex", 51 | ] 52 | 53 | [[package]] 54 | name = "tree-sitter-repro" 55 | version = "0.1.0" 56 | dependencies = [ 57 | "cc", 58 | "tree-sitter", 59 | ] 60 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tree-sitter-repro" 3 | description = "repro grammar for the tree-sitter parsing library" 4 | version = "0.1.0" 5 | repository = "https://github.com/sourcegraph/scip/cmd/scip/tests/reprolang" 6 | edition = "2021" 7 | license = "Apache-2.0" 8 | 9 | build = "bindings/rust/build.rs" 10 | include = [ 11 | "bindings/rust/*", 12 | "grammar.js", 13 | "queries/*", 14 | "src/*", 15 | ] 16 | 17 | rust-version = "1.81.0" 18 | 19 | [lib] 20 | path = "bindings/rust/lib.rs" 21 | 22 | [dependencies] 23 | tree-sitter = "~0.20" 24 | 25 | [build-dependencies] 26 | cc = "1.0" 27 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/binding.gyp: -------------------------------------------------------------------------------- 1 | { 2 | "targets": [ 3 | { 4 | "target_name": "tree_sitter_reprolang_binding", 5 | "include_dirs": [ 6 | "= 0 { 38 | continue 39 | } 40 | documentation := []string{"signature of " + def.name.value} 41 | if def.docstring != "" { 42 | documentation = append(documentation, def.docstring) 43 | } 44 | result = append(result, &scip.SymbolInformation{ 45 | Symbol: def.name.symbol, 46 | Documentation: documentation, 47 | Relationships: def.relations.toSCIP(), 48 | }) 49 | } 50 | // Ensure a stable order of relationships 51 | sort.SliceStable(result, func(i, j int) bool { 52 | return result[i].Symbol < result[j].Symbol 53 | }) 54 | return result 55 | } 56 | 57 | func (s *reproSourceFile) occurrences() []*scip.Occurrence { 58 | var result []*scip.Occurrence 59 | emit := func(rel relationships) { 60 | for _, ident := range rel.identifiers() { 61 | if ident == nil { 62 | continue 63 | } 64 | result = append(result, ident.occurrence(scip.SymbolRole_UnspecifiedSymbolRole)) 65 | } 66 | } 67 | for _, def := range s.definitions { 68 | result = append(result, def.name.occurrence(scip.SymbolRole_Definition)) 69 | emit(def.relations) 70 | } 71 | for _, rel := range s.relationships { 72 | emit(rel.relations) 73 | } 74 | for _, ref := range s.references { 75 | role := scip.SymbolRole_UnspecifiedSymbolRole 76 | if ref.isForwardDef { 77 | role = scip.SymbolRole_ForwardDefinition 78 | } 79 | result = append(result, ref.name.occurrence(role)) 80 | } 81 | return result 82 | } 83 | 84 | func (r *relationships) toSCIP() []*scip.Relationship { 85 | bySymbol := map[string]*scip.Relationship{} 86 | for _, ident := range r.identifiers() { 87 | if ident == nil { 88 | continue 89 | } 90 | bySymbol[ident.symbol] = &scip.Relationship{Symbol: ident.symbol} 91 | } 92 | if r.implementsRelation != nil { 93 | bySymbol[r.implementsRelation.symbol].IsImplementation = true 94 | } 95 | if r.referencesRelation != nil { 96 | bySymbol[r.referencesRelation.symbol].IsReference = true 97 | } 98 | if r.typeDefinesRelation != nil { 99 | bySymbol[r.typeDefinesRelation.symbol].IsTypeDefinition = true 100 | } 101 | if r.definedByRelation != nil { 102 | bySymbol[r.definedByRelation.symbol].IsDefinition = true 103 | } 104 | var result []*scip.Relationship 105 | for _, value := range bySymbol { 106 | result = append(result, value) 107 | } 108 | return result 109 | } 110 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/bindings/go/repro/source_file.go: -------------------------------------------------------------------------------- 1 | package repro 2 | 3 | import ( 4 | sitter "github.com/smacker/go-tree-sitter" 5 | 6 | "github.com/sourcegraph/scip/bindings/go/scip" 7 | ) 8 | 9 | type reproSourceFile struct { 10 | Source *scip.SourceFile 11 | node *sitter.Node 12 | definitions []*definitionStatement 13 | references []*referenceStatement 14 | relationships []*relationshipsStatement 15 | localScope *reproScope 16 | } 17 | 18 | func newSourceFile(sourceFile *scip.SourceFile, node *sitter.Node) *reproSourceFile { 19 | return &reproSourceFile{ 20 | Source: sourceFile, 21 | node: node, 22 | definitions: nil, 23 | references: nil, 24 | localScope: newScope(), 25 | } 26 | } 27 | 28 | func (s *reproSourceFile) nodeText(n *sitter.Node) string { 29 | return s.Source.Text[n.StartByte():n.EndByte()] 30 | } 31 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/bindings/node/binding.cc: -------------------------------------------------------------------------------- 1 | #include "tree_sitter/parser.h" 2 | #include 3 | #include "nan.h" 4 | 5 | using namespace v8; 6 | 7 | extern "C" TSLanguage * tree_sitter_reprolang(); 8 | 9 | namespace { 10 | 11 | NAN_METHOD(New) {} 12 | 13 | void Init(Local exports, Local module) { 14 | Local tpl = Nan::New(New); 15 | tpl->SetClassName(Nan::New("Language").ToLocalChecked()); 16 | tpl->InstanceTemplate()->SetInternalFieldCount(1); 17 | 18 | Local constructor = Nan::GetFunction(tpl).ToLocalChecked(); 19 | Local instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked(); 20 | Nan::SetInternalFieldPointer(instance, 0, tree_sitter_reprolang()); 21 | 22 | Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("reprolang").ToLocalChecked()); 23 | Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance); 24 | } 25 | 26 | NODE_MODULE(tree_sitter_reprolang_binding, Init) 27 | 28 | } // namespace 29 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/bindings/node/index.js: -------------------------------------------------------------------------------- 1 | try { 2 | module.exports = require('../../build/Release/tree_sitter_reprolang_binding') 3 | } catch (error1) { 4 | if (error1.code !== 'MODULE_NOT_FOUND') { 5 | throw error1 6 | } 7 | try { 8 | module.exports = require('../../build/Debug/tree_sitter_reprolang_binding') 9 | } catch (error2) { 10 | if (error2.code !== 'MODULE_NOT_FOUND') { 11 | throw error2 12 | } 13 | throw error1 14 | } 15 | } 16 | 17 | try { 18 | module.exports.nodeTypeInfo = require('../../src/node-types.json') 19 | } catch (_) {} 20 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/bindings/rust/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let src_dir = std::path::Path::new("src"); 3 | 4 | let mut c_config = cc::Build::new(); 5 | c_config.include(&src_dir); 6 | c_config 7 | .flag_if_supported("-Wno-unused-parameter") 8 | .flag_if_supported("-Wno-unused-but-set-variable") 9 | .flag_if_supported("-Wno-trigraphs"); 10 | let parser_path = src_dir.join("parser.c"); 11 | c_config.file(&parser_path); 12 | 13 | // If your language uses an external scanner written in C, 14 | // then include this block of code: 15 | 16 | /* 17 | let scanner_path = src_dir.join("scanner.c"); 18 | c_config.file(&scanner_path); 19 | println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); 20 | */ 21 | 22 | c_config.compile("parser"); 23 | println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap()); 24 | 25 | // If your language uses an external scanner written in C++, 26 | // then include this block of code: 27 | 28 | /* 29 | let mut cpp_config = cc::Build::new(); 30 | cpp_config.cpp(true); 31 | cpp_config.include(&src_dir); 32 | cpp_config 33 | .flag_if_supported("-Wno-unused-parameter") 34 | .flag_if_supported("-Wno-unused-but-set-variable"); 35 | let scanner_path = src_dir.join("scanner.cc"); 36 | cpp_config.file(&scanner_path); 37 | cpp_config.compile("scanner"); 38 | println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); 39 | */ 40 | } 41 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/bindings/rust/lib.rs: -------------------------------------------------------------------------------- 1 | //! This crate provides reprolang language support for the [tree-sitter][] parsing library. 2 | //! 3 | //! Typically, you will use the [language][language func] function to add this language to a 4 | //! tree-sitter [Parser][], and then use the parser to parse some code: 5 | //! 6 | //! ``` 7 | //! let code = ""; 8 | //! let mut parser = tree_sitter::Parser::new(); 9 | //! parser.set_language(tree_sitter_reprolang::language()).expect("Error loading reprolang grammar"); 10 | //! let tree = parser.parse(code, None).unwrap(); 11 | //! ``` 12 | //! 13 | //! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html 14 | //! [language func]: fn.language.html 15 | //! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html 16 | //! [tree-sitter]: https://tree-sitter.github.io/ 17 | 18 | use tree_sitter::Language; 19 | 20 | extern "C" { 21 | fn tree_sitter_reprolang() -> Language; 22 | } 23 | 24 | /// Get the tree-sitter [Language][] for this grammar. 25 | /// 26 | /// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html 27 | pub fn language() -> Language { 28 | unsafe { tree_sitter_reprolang() } 29 | } 30 | 31 | /// The content of the [`node-types.json`][] file for this grammar. 32 | /// 33 | /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types 34 | pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json"); 35 | 36 | // Uncomment these to include any queries that this grammar contains 37 | 38 | // pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm"); 39 | // pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm"); 40 | // pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm"); 41 | // pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm"); 42 | 43 | #[cfg(test)] 44 | mod tests { 45 | #[test] 46 | fn test_can_load_grammar() { 47 | let mut parser = tree_sitter::Parser::new(); 48 | parser 49 | .set_language(super::language()) 50 | .expect("Error loading reprolang language"); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/generate-tree-sitter-parser.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eux 3 | SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" 4 | cd "$SCRIPT_DIR" 5 | yarn install 6 | 7 | # See https://github.com/smacker/go-tree-sitter/issues/85#issuecomment-1287988200 8 | ./node_modules/.bin/tree-sitter generate --abi 13 9 | 10 | yarn --cwd ../../../.. run prettier 11 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/grammar.js: -------------------------------------------------------------------------------- 1 | module.exports = grammar({ 2 | name: 'reprolang', 3 | extras: $ => [/\s+/], 4 | word: $ => $.workspace_identifier, 5 | 6 | rules: { 7 | source_file: $ => repeat($._statement), 8 | _statement: $ => 9 | seq( 10 | choice( 11 | $.definition_statement, 12 | $.reference_statement, 13 | $.relationships_statement, 14 | $.comment 15 | ), 16 | '\n' 17 | ), 18 | definition_statement: $ => 19 | seq( 20 | field('docstring', optional(seq($.docstring, '\n'))), 21 | 'definition', 22 | field('name', $.identifier), 23 | field('roles', repeat($._definition_relations)) 24 | ), 25 | reference_statement: $ => 26 | seq( 27 | 'reference', 28 | field('forward_definition', optional('forward_definition')), 29 | field('name', $.identifier) 30 | ), 31 | _definition_relations: $ => 32 | choice( 33 | $.implementation_relation, 34 | $.type_definition_relation, 35 | $.references_relation 36 | ), 37 | implementation_relation: $ => 38 | seq('implements', field('name', $.identifier)), 39 | type_definition_relation: $ => 40 | seq('type_defines', field('name', $.identifier)), 41 | references_relation: $ => seq('references', field('name', $.identifier)), 42 | // Meant to be used primarily when trying to construct indexes with 43 | // relationships for symbols which lack a definition themselves, 44 | // and are defined by some other symbol. 45 | relationships_statement: $ => 46 | seq( 47 | 'relationships', 48 | field('name', $.identifier), 49 | field('roles', repeat($._all_relations)) 50 | ), 51 | _all_relations: $ => choice($._definition_relations, $.defined_by_relation), 52 | defined_by_relation: $ => seq('defined_by', field('name', $.identifier)), 53 | comment: $ => seq('#', /.*/), 54 | docstring: $ => seq('# docstring:', /.*/), 55 | identifier: $ => 56 | choice( 57 | field('global', $.global_identifier), 58 | field('workspace', $.workspace_identifier) 59 | ), 60 | global_identifier: $ => 61 | seq( 62 | 'global', 63 | field('project_name', $.workspace_identifier), 64 | field('descriptors', $.workspace_identifier) 65 | ), 66 | workspace_identifier: $ => /[^\s]+/, 67 | }, 68 | }) 69 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "reprolang", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "bindings/node", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "nan": "^2.22.0" 13 | }, 14 | "devDependencies": { 15 | "tree-sitter-cli": "0.21.0" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/src/binding.go: -------------------------------------------------------------------------------- 1 | package src 2 | 3 | //#include "tree_sitter/parser.h" 4 | //TSLanguage *tree_sitter_reprolang(); 5 | import "C" 6 | 7 | import ( 8 | "unsafe" 9 | 10 | sitter "github.com/smacker/go-tree-sitter" 11 | ) 12 | 13 | func GetLanguage() *sitter.Language { 14 | ptr := unsafe.Pointer(C.tree_sitter_reprolang()) 15 | return sitter.NewLanguage(ptr) 16 | } 17 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/src/binding_test.go: -------------------------------------------------------------------------------- 1 | package src 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | sitter "github.com/smacker/go-tree-sitter" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestParse(t *testing.T) { 12 | assert := assert.New(t) 13 | 14 | n, err := sitter.ParseCtx(context.Background(), []byte("definition a implements b\n"), GetLanguage()) 15 | assert.NoError(err) 16 | assert.Equal( 17 | "(source_file (definition_statement name: (identifier workspace: (workspace_identifier)) roles: (implementation_relation name: (identifier workspace: (workspace_identifier)))))", 18 | n.String(), 19 | ) 20 | } 21 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/src/tree_sitter/parser.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_PARSER_H_ 2 | #define TREE_SITTER_PARSER_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #define ts_builtin_sym_error ((TSSymbol)-1) 13 | #define ts_builtin_sym_end 0 14 | #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 15 | 16 | #ifndef TREE_SITTER_API_H_ 17 | typedef uint16_t TSStateId; 18 | typedef uint16_t TSSymbol; 19 | typedef uint16_t TSFieldId; 20 | typedef struct TSLanguage TSLanguage; 21 | #endif 22 | 23 | typedef struct { 24 | TSFieldId field_id; 25 | uint8_t child_index; 26 | bool inherited; 27 | } TSFieldMapEntry; 28 | 29 | typedef struct { 30 | uint16_t index; 31 | uint16_t length; 32 | } TSFieldMapSlice; 33 | 34 | typedef struct { 35 | bool visible; 36 | bool named; 37 | bool supertype; 38 | } TSSymbolMetadata; 39 | 40 | typedef struct TSLexer TSLexer; 41 | 42 | struct TSLexer { 43 | int32_t lookahead; 44 | TSSymbol result_symbol; 45 | void (*advance)(TSLexer *, bool); 46 | void (*mark_end)(TSLexer *); 47 | uint32_t (*get_column)(TSLexer *); 48 | bool (*is_at_included_range_start)(const TSLexer *); 49 | bool (*eof)(const TSLexer *); 50 | }; 51 | 52 | typedef enum { 53 | TSParseActionTypeShift, 54 | TSParseActionTypeReduce, 55 | TSParseActionTypeAccept, 56 | TSParseActionTypeRecover, 57 | } TSParseActionType; 58 | 59 | typedef union { 60 | struct { 61 | uint8_t type; 62 | TSStateId state; 63 | bool extra; 64 | bool repetition; 65 | } shift; 66 | struct { 67 | uint8_t type; 68 | uint8_t child_count; 69 | TSSymbol symbol; 70 | int16_t dynamic_precedence; 71 | uint16_t production_id; 72 | } reduce; 73 | uint8_t type; 74 | } TSParseAction; 75 | 76 | typedef struct { 77 | uint16_t lex_state; 78 | uint16_t external_lex_state; 79 | } TSLexMode; 80 | 81 | typedef union { 82 | TSParseAction action; 83 | struct { 84 | uint8_t count; 85 | bool reusable; 86 | } entry; 87 | } TSParseActionEntry; 88 | 89 | struct TSLanguage { 90 | uint32_t version; 91 | uint32_t symbol_count; 92 | uint32_t alias_count; 93 | uint32_t token_count; 94 | uint32_t external_token_count; 95 | uint32_t state_count; 96 | uint32_t large_state_count; 97 | uint32_t production_id_count; 98 | uint32_t field_count; 99 | uint16_t max_alias_sequence_length; 100 | const uint16_t *parse_table; 101 | const uint16_t *small_parse_table; 102 | const uint32_t *small_parse_table_map; 103 | const TSParseActionEntry *parse_actions; 104 | const char * const *symbol_names; 105 | const char * const *field_names; 106 | const TSFieldMapSlice *field_map_slices; 107 | const TSFieldMapEntry *field_map_entries; 108 | const TSSymbolMetadata *symbol_metadata; 109 | const TSSymbol *public_symbol_map; 110 | const uint16_t *alias_map; 111 | const TSSymbol *alias_sequences; 112 | const TSLexMode *lex_modes; 113 | bool (*lex_fn)(TSLexer *, TSStateId); 114 | bool (*keyword_lex_fn)(TSLexer *, TSStateId); 115 | TSSymbol keyword_capture_token; 116 | struct { 117 | const bool *states; 118 | const TSSymbol *symbol_map; 119 | void *(*create)(void); 120 | void (*destroy)(void *); 121 | bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); 122 | unsigned (*serialize)(void *, char *); 123 | void (*deserialize)(void *, const char *, unsigned); 124 | } external_scanner; 125 | const TSStateId *primary_state_ids; 126 | }; 127 | 128 | /* 129 | * Lexer Macros 130 | */ 131 | 132 | #ifdef _MSC_VER 133 | #define UNUSED __pragma(warning(suppress : 4101)) 134 | #else 135 | #define UNUSED __attribute__((unused)) 136 | #endif 137 | 138 | #define START_LEXER() \ 139 | bool result = false; \ 140 | bool skip = false; \ 141 | UNUSED \ 142 | bool eof = false; \ 143 | int32_t lookahead; \ 144 | goto start; \ 145 | next_state: \ 146 | lexer->advance(lexer, skip); \ 147 | start: \ 148 | skip = false; \ 149 | lookahead = lexer->lookahead; 150 | 151 | #define ADVANCE(state_value) \ 152 | { \ 153 | state = state_value; \ 154 | goto next_state; \ 155 | } 156 | 157 | #define SKIP(state_value) \ 158 | { \ 159 | skip = true; \ 160 | state = state_value; \ 161 | goto next_state; \ 162 | } 163 | 164 | #define ACCEPT_TOKEN(symbol_value) \ 165 | result = true; \ 166 | lexer->result_symbol = symbol_value; \ 167 | lexer->mark_end(lexer); 168 | 169 | #define END_STATE() return result; 170 | 171 | /* 172 | * Parse Table Macros 173 | */ 174 | 175 | #define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) 176 | 177 | #define STATE(id) id 178 | 179 | #define ACTIONS(id) id 180 | 181 | #define SHIFT(state_value) \ 182 | {{ \ 183 | .shift = { \ 184 | .type = TSParseActionTypeShift, \ 185 | .state = (state_value) \ 186 | } \ 187 | }} 188 | 189 | #define SHIFT_REPEAT(state_value) \ 190 | {{ \ 191 | .shift = { \ 192 | .type = TSParseActionTypeShift, \ 193 | .state = (state_value), \ 194 | .repetition = true \ 195 | } \ 196 | }} 197 | 198 | #define SHIFT_EXTRA() \ 199 | {{ \ 200 | .shift = { \ 201 | .type = TSParseActionTypeShift, \ 202 | .extra = true \ 203 | } \ 204 | }} 205 | 206 | #define REDUCE(symbol_val, child_count_val, ...) \ 207 | {{ \ 208 | .reduce = { \ 209 | .type = TSParseActionTypeReduce, \ 210 | .symbol = symbol_val, \ 211 | .child_count = child_count_val, \ 212 | __VA_ARGS__ \ 213 | }, \ 214 | }} 215 | 216 | #define RECOVER() \ 217 | {{ \ 218 | .type = TSParseActionTypeRecover \ 219 | }} 220 | 221 | #define ACCEPT_INPUT() \ 222 | {{ \ 223 | .type = TSParseActionTypeAccept \ 224 | }} 225 | 226 | #ifdef __cplusplus 227 | } 228 | #endif 229 | 230 | #endif // TREE_SITTER_PARSER_H_ 231 | -------------------------------------------------------------------------------- /cmd/scip/tests/reprolang/src/workaround.go: -------------------------------------------------------------------------------- 1 | package src 2 | 3 | // This function exists only to make lsif-go succeed 4 | // Without this function, lsif-go fails with the error message 5 | // error: failed to index: failed to load packages: packages.Load: err: exit status 1: stderr: go build github.com/sourcegraph/sourcegraph/lib/codeintel/reprolang/src: build constraints exclude all Go files in /__w/sourcegraph/sourcegraph/lib/codeintel/reprolang/src 6 | // The error appears related to the usage of CGO in the sibling file binding.go https://github.com/golang/go/issues/24068 7 | // By adding this file, there exists at least one non-test file in this package that doesn't use CGO. 8 | //lint:ignore U1000 This function is intentionally unused 9 | func lsifWorkaround() {} 10 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/cyclic-reference/cycle1.repro: -------------------------------------------------------------------------------- 1 | # Test cyclic references between files. 2 | definition hello(). 3 | reference hello(). 4 | reference hello2(). 5 | 6 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/cyclic-reference/cycle2.repro: -------------------------------------------------------------------------------- 1 | # Test cyclic references between files. 2 | definition hello2(). 3 | reference hello(). 4 | reference hello2(). 5 | 6 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/diagnostics/diagnostics.repro: -------------------------------------------------------------------------------- 1 | definition deprecatedMethod. 2 | reference deprecatedMethod. 3 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/duplicates/duplicate.repro: -------------------------------------------------------------------------------- 1 | definition readFileSync. 2 | definition readFileSync. 3 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/forward-def/forward_def.repro: -------------------------------------------------------------------------------- 1 | reference forward_definition abc# 2 | definition abc# 3 | reference abc# 4 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/global-cross-repo/reference.repro: -------------------------------------------------------------------------------- 1 | # Reference a global symbol from another workspace. 2 | reference global global-workspace hello.repro/hello(). 3 | reference global duplicates duplicate.repro/readFileSync. 4 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/global-workspace/hello.repro: -------------------------------------------------------------------------------- 1 | definition hello(). -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/implementation-cross-repo/bird.repro: -------------------------------------------------------------------------------- 1 | # Test how to implement a symbol from an external workspace. 2 | definition bird# implements global implementation animal.repro/animal# 3 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/implementation/animal.repro: -------------------------------------------------------------------------------- 1 | # Test how to implement a symbol within the same workspace. 2 | definition animal# 3 | definition dog# implements animal# 4 | definition cat# implements animal# 5 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/local-document/local1.repro: -------------------------------------------------------------------------------- 1 | # docstring: local is a local method 2 | definition localExample 3 | reference localExample 4 | 5 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/local-document/local2.repro: -------------------------------------------------------------------------------- 1 | definition localExample 2 | reference localExample -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/missing-symbol-information/globals.repro: -------------------------------------------------------------------------------- 1 | definition NoSymbolInformation# 2 | reference NoSymbolInformation# -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/missing-symbol-information/locals.repro: -------------------------------------------------------------------------------- 1 | definition localNoSymbolInformation 2 | reference localNoSymbolInformation -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/relationships/defined_by.repro: -------------------------------------------------------------------------------- 1 | definition M_f. 2 | 3 | definition C1_f. 4 | 5 | reference C2_f. 6 | 7 | relationships C2_f. defined_by C1_f. references M_f. 8 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/relationships/mixed.repro: -------------------------------------------------------------------------------- 1 | definition local1 2 | definition local2 3 | definition local3 4 | definition local4 implements local1 references local2 type_defines local3 5 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/relationships/references.repro: -------------------------------------------------------------------------------- 1 | definition local1 2 | definition local2 references local1 3 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/input/relationships/type_defines.repro: -------------------------------------------------------------------------------- 1 | definition local1 2 | definition local2 type_defines local1 3 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/cyclic-reference/cycle1.repro: -------------------------------------------------------------------------------- 1 | # Test cyclic references between files. 2 | definition hello(). 3 | # ^^^^^^^^ definition cycle1.repro/hello(). 4 | # documentation 5 | # > signature of hello(). 6 | reference hello(). 7 | # ^^^^^^^^ reference cycle1.repro/hello(). 8 | reference hello2(). 9 | # ^^^^^^^^^ reference cycle2.repro/hello2(). 10 | 11 | 12 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/cyclic-reference/cycle2.repro: -------------------------------------------------------------------------------- 1 | # Test cyclic references between files. 2 | definition hello2(). 3 | # ^^^^^^^^^ definition cycle2.repro/hello2(). 4 | # documentation 5 | # > signature of hello2(). 6 | reference hello(). 7 | # ^^^^^^^^ reference cycle1.repro/hello(). 8 | reference hello2(). 9 | # ^^^^^^^^^ reference cycle2.repro/hello2(). 10 | 11 | 12 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/diagnostics/diagnostics.repro: -------------------------------------------------------------------------------- 1 | definition deprecatedMethod. 2 | # ^^^^^^^^^^^^^^^^^ definition diagnostics.repro/deprecatedMethod. 3 | # documentation 4 | # > signature of deprecatedMethod. 5 | # diagnostic Warning: 6 | # > deprecated identifier 7 | reference deprecatedMethod. 8 | # ^^^^^^^^^^^^^^^^^ reference diagnostics.repro/deprecatedMethod. 9 | # diagnostic Warning: 10 | # > deprecated identifier 11 | 12 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/duplicates/duplicate.repro: -------------------------------------------------------------------------------- 1 | definition readFileSync. 2 | # ^^^^^^^^^^^^^ definition duplicate.repro/readFileSync. 3 | # documentation 4 | # > signature of readFileSync. 5 | definition readFileSync. 6 | # ^^^^^^^^^^^^^ definition duplicate.repro/readFileSync. 7 | # documentation 8 | # > signature of readFileSync. 9 | 10 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/forward-def/forward_def.repro: -------------------------------------------------------------------------------- 1 | reference forward_definition abc# 2 | # ^^^^ forward_definition forward_def.repro/abc# 3 | definition abc# 4 | # ^^^^ definition forward_def.repro/abc# 5 | # documentation 6 | # > signature of abc# 7 | reference abc# 8 | # ^^^^ reference forward_def.repro/abc# 9 | 10 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/global-cross-repo/reference.repro: -------------------------------------------------------------------------------- 1 | # Reference a global symbol from another workspace. 2 | reference global global-workspace hello.repro/hello(). 3 | # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reference global-workspace hello.repro/hello(). 4 | reference global duplicates duplicate.repro/readFileSync. 5 | # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reference duplicates duplicate.repro/readFileSync. 6 | 7 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/global-workspace/hello.repro: -------------------------------------------------------------------------------- 1 | definition hello(). 2 | # ^^^^^^^^ definition hello.repro/hello(). 3 | # documentation 4 | # > signature of hello(). 5 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/implementation-cross-repo/bird.repro: -------------------------------------------------------------------------------- 1 | # Test how to implement a symbol from an external workspace. 2 | definition bird# implements global implementation animal.repro/animal# 3 | # ^^^^^ definition bird.repro/bird# 4 | # documentation 5 | # > signature of bird# 6 | # relationship implementation animal.repro/animal# implementation 7 | # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reference implementation animal.repro/animal# 8 | 9 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/implementation/animal.repro: -------------------------------------------------------------------------------- 1 | # Test how to implement a symbol within the same workspace. 2 | definition animal# 3 | # ^^^^^^^ definition animal.repro/animal# 4 | # documentation 5 | # > signature of animal# 6 | definition dog# implements animal# 7 | # ^^^^ definition animal.repro/dog# 8 | # documentation 9 | # > signature of dog# 10 | # relationship animal.repro/animal# implementation 11 | # ^^^^^^^ reference animal.repro/animal# 12 | definition cat# implements animal# 13 | # ^^^^ definition animal.repro/cat# 14 | # documentation 15 | # > signature of cat# 16 | # relationship animal.repro/animal# implementation 17 | # ^^^^^^^ reference animal.repro/animal# 18 | 19 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/local-document/local1.repro: -------------------------------------------------------------------------------- 1 | # docstring: local is a local method 2 | definition localExample 3 | # ^^^^^^^^^^^^ definition local Example 4 | # documentation 5 | # > signature of localExample 6 | # documentation 7 | # > : local is a local method 8 | reference localExample 9 | # ^^^^^^^^^^^^ reference local Example 10 | 11 | 12 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/local-document/local2.repro: -------------------------------------------------------------------------------- 1 | definition localExample 2 | # ^^^^^^^^^^^^ definition local Example 3 | # documentation 4 | # > signature of localExample 5 | reference localExample 6 | # ^^^^^^^^^^^^ reference local Example 7 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/missing-symbol-information/globals.repro: -------------------------------------------------------------------------------- 1 | definition NoSymbolInformation# 2 | # ^^^^^^^^^^^^^^^^^^^^ definition globals.repro/NoSymbolInformation# 3 | reference NoSymbolInformation# 4 | # ^^^^^^^^^^^^^^^^^^^^ reference globals.repro/NoSymbolInformation# 5 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/missing-symbol-information/locals.repro: -------------------------------------------------------------------------------- 1 | definition localNoSymbolInformation 2 | # ^^^^^^^^^^^^^^^^^^^^^^^^ definition local NoSymbolInformation 3 | reference localNoSymbolInformation 4 | # ^^^^^^^^^^^^^^^^^^^^^^^^ reference local NoSymbolInformation 5 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/relationships/defined_by.repro: -------------------------------------------------------------------------------- 1 | definition M_f. 2 | # ^^^^ definition defined_by.repro/M_f. 3 | # documentation 4 | # > signature of M_f. 5 | 6 | definition C1_f. 7 | # ^^^^^ definition defined_by.repro/C1_f. 8 | # documentation 9 | # > signature of C1_f. 10 | 11 | reference C2_f. 12 | # ^^^^^ reference defined_by.repro/C1_f. 13 | 14 | relationships C2_f. defined_by C1_f. references M_f. 15 | # ^^^^^ reference defined_by.repro/C1_f. 16 | # ^^^^ reference defined_by.repro/M_f. 17 | 18 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/relationships/mixed.repro: -------------------------------------------------------------------------------- 1 | definition local1 2 | # ^^^^^^ definition local 1 3 | # documentation 4 | # > signature of local1 5 | definition local2 6 | # ^^^^^^ definition local 2 7 | # documentation 8 | # > signature of local2 9 | definition local3 10 | # ^^^^^^ definition local 3 11 | # documentation 12 | # > signature of local3 13 | definition local4 implements local1 references local2 type_defines local3 14 | # ^^^^^^ definition local 4 15 | # documentation 16 | # > signature of local4 17 | # relationship local 1 implementation 18 | # relationship local 2 reference 19 | # relationship local 3 type_definition 20 | # ^^^^^^ reference local 1 21 | # ^^^^^^ reference local 2 22 | # ^^^^^^ reference local 3 23 | 24 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/relationships/references.repro: -------------------------------------------------------------------------------- 1 | definition local1 2 | # ^^^^^^ definition local 1 3 | # documentation 4 | # > signature of local1 5 | definition local2 references local1 6 | # ^^^^^^ definition local 2 7 | # documentation 8 | # > signature of local2 9 | # relationship local 1 reference 10 | # ^^^^^^ reference local 1 11 | 12 | -------------------------------------------------------------------------------- /cmd/scip/tests/snapshots/output/relationships/type_defines.repro: -------------------------------------------------------------------------------- 1 | definition local1 2 | # ^^^^^^ definition local 1 3 | # documentation 4 | # > signature of local1 5 | definition local2 type_defines local1 6 | # ^^^^^^ definition local 2 7 | # documentation 8 | # > signature of local2 9 | # relationship local 1 type_definition 10 | # ^^^^^^ reference local 1 11 | 12 | -------------------------------------------------------------------------------- /cmd/scip/tests/test_cmd/diagnostics/fails-incorrect-diagnostic.repro: -------------------------------------------------------------------------------- 1 | definition deprecatedMethod. 2 | # ^ diagnostic Warning: 3 | # > THIS IS NOT CORRECT -------------------------------------------------------------------------------- /cmd/scip/tests/test_cmd/diagnostics/fails-no-diagnostic.repro: -------------------------------------------------------------------------------- 1 | definition hello(). 2 | # ^ diagnostic Warning: 3 | # > deprecated identifier 4 | -------------------------------------------------------------------------------- /cmd/scip/tests/test_cmd/diagnostics/passes.repro: -------------------------------------------------------------------------------- 1 | definition deprecatedMethod. 2 | # ^ diagnostic Warning 3 | # > deprecated identifier 4 | 5 | reference deprecatedMethod. 6 | # ^ diagnostic Warning 7 | # > deprecated identifier 8 | 9 | -------------------------------------------------------------------------------- /cmd/scip/tests/test_cmd/ranges/fails.repro: -------------------------------------------------------------------------------- 1 | definition hello(). 2 | # ^ definition passes.repro/hello(). -------------------------------------------------------------------------------- /cmd/scip/tests/test_cmd/ranges/passes.repro: -------------------------------------------------------------------------------- 1 | definition hello(). 2 | # ^^^^^^^^ definition reprolang repro_manager ranges 1.0.0 passes.repro/hello(). 3 | # ^ definition reprolang repro_manager ranges 1.0.0 passes.repro/hello(). 4 | # <- definition reprolang repro_manager ranges 1.0.0 passes.repro/hello(). -------------------------------------------------------------------------------- /cmd/scip/tests/test_cmd/roles/fails-wrong-role.repro: -------------------------------------------------------------------------------- 1 | definition hello(). 2 | # ^ reference reprolang repro_manager roles 1.0.0 fails-wrong-role.repro/hello(). -------------------------------------------------------------------------------- /cmd/scip/tests/test_cmd/roles/fails-wrong-symbol.repro: -------------------------------------------------------------------------------- 1 | definition hello(). 2 | # ^ definition reprolang repro_manager roles 1.0.0 fails-wrong-role.repro/hello2(). -------------------------------------------------------------------------------- /cmd/scip/tests/test_cmd/roles/passes.repro: -------------------------------------------------------------------------------- 1 | definition hello(). 2 | # ^ definition reprolang repro_manager roles 1.0.0 passes.repro/hello(). 3 | 4 | reference hello(). 5 | # ^ reference reprolang repro_manager roles 1.0.0 passes.repro/hello(). 6 | 7 | definition abc# 8 | reference forward_definition abc# 9 | # ^ forward_definition reprolang repro_manager roles 1.0.0 passes.repro/abc# -------------------------------------------------------------------------------- /cmd/scip/version.txt: -------------------------------------------------------------------------------- 1 | 0.5.2 2 | -------------------------------------------------------------------------------- /dev/Dockerfile.bindings: -------------------------------------------------------------------------------- 1 | FROM ubuntu@sha256:19478ce7fc2ffbce89df29fea5725a8d12e57de52eb9ea570890dc5852aac1ac 2 | 3 | ENV DEBIAN_FRONTEND noninteractive 4 | 5 | RUN apt-get update -q && apt-get install -y git curl xz-utils tar gpg build-essential libssl-dev zlib1g-dev \ 6 | libbz2-dev libreadline-dev libsqlite3-dev \ 7 | libncursesw5-dev tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ 8 | libffi8 libgmp-dev libgmp10 libncurses-dev libncurses5 libtinfo5 pkg-config 9 | 10 | RUN groupadd -g 1001 asdf 11 | RUN adduser -u 1001 --gid 1001 --shell /bin/bash --home /asdf --disabled-password asdf 12 | ENV PATH="${PATH}:/asdf/.asdf/shims:/asdf/.asdf/bin" 13 | 14 | USER asdf 15 | RUN git clone --depth 1 https://github.com/asdf-vm/asdf.git $HOME/.asdf && \ 16 | echo '. $HOME/.asdf/asdf.sh' >> $HOME/.bashrc && \ 17 | echo '. $HOME/.asdf/asdf.sh' >> $HOME/.profile 18 | 19 | RUN . ~/.bashrc 20 | 21 | # YOLO 22 | RUN curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh 23 | 24 | ENV PATH="${PATH}:/asdf/.ghcup/bin" 25 | 26 | RUN asdf plugin add nodejs && \ 27 | asdf plugin add golang && \ 28 | asdf plugin add shellcheck && \ 29 | asdf plugin add yarn && \ 30 | asdf plugin add python && \ 31 | asdf plugin add rust 32 | 33 | COPY .tool-versions . 34 | 35 | RUN asdf install && \ 36 | # Pre-fetch Haskell dependencies as they take the longest time by far 37 | # TODO(anton): run the proto-generate.sh script during the build time to 38 | # pre-fecth all dependencies. I attempted to do so but ran into issues 39 | # with Yarn 40 | cabal install proto-lens-protoc-0.8.0.1 ghc-source-gen-0.4.5.0 41 | 42 | WORKDIR /src 43 | 44 | CMD ["./dev/docker-entrypoint.sh"] 45 | 46 | 47 | -------------------------------------------------------------------------------- /dev/build-docker-environment.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -xeuo pipefail 4 | 5 | cd "$(dirname "${BASH_SOURCE[0]}")/.." # cd to repo root dir 6 | 7 | IMAGE_NAME="sourcegraph/scip-bindings-env" 8 | 9 | docker build . -t $IMAGE_NAME -f ./dev/Dockerfile.bindings --cache-from ghcr.io/$IMAGE_NAME:latest && echo $IMAGE_NAME 10 | -------------------------------------------------------------------------------- /dev/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -xeuo pipefail 4 | 5 | ./dev/proto-generate.sh 6 | 7 | ./cmd/scip/tests/reprolang/generate-tree-sitter-parser.sh 8 | -------------------------------------------------------------------------------- /dev/generate-all-in-docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -xeuo pipefail 4 | 5 | cd "$(dirname "${BASH_SOURCE[0]}")/.." # cd to repo root dir 6 | 7 | IMAGE_NAME=$(./dev/build-docker-environment.sh) 8 | 9 | docker run -v $(pwd):/src $IMAGE_NAME 10 | 11 | -------------------------------------------------------------------------------- /dev/proto-generate.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -xeuo pipefail 4 | 5 | cd "$(dirname "${BASH_SOURCE[0]}")/.." # cd to repo root dir 6 | 7 | echo "--- yarn in root ---" 8 | yarn --cwd ./bindings/typescript install --frozen-lockfile 9 | 10 | echo "--- cargo install rust-protobuf ---" 11 | PROTOC_GEN_RUST_VERSION="$(cat bindings/rust/Cargo.toml | grep 'protobuf =' | sed -E 's/.*\"=(.+)\"/\1/g')" 12 | if ! grep -q "$PROTOC_GEN_RUST_VERSION" "./.bin/PROTOC_GEN_RUST_VERSION" \ 13 | || ! test -f "./.bin/bin/protoc-gen-rs"; then 14 | rm -rf .bin 15 | cargo install --root .bin protobuf-codegen --version 3.7.2 --locked 16 | echo "$PROTOC_GEN_RUST_VERSION" > "./.bin/PROTOC_GEN_RUST_VERSION" 17 | fi 18 | 19 | 20 | echo "--- Haskell ---" 21 | command -v cabal > /dev/null 2>&1 || { echo >&2 "Haskell's 'cabal' command is not installed. Please install it first via https://www.haskell.org/ghcup/"; exit 1; } 22 | cabal install proto-lens-protoc-0.8.0.1 ghc-source-gen-0.4.5.0 --overwrite-policy=always --ghc-options='-j2 +RTS -A32m' --installdir="$PWD/.bin" 23 | # buf requires the generator to be named protoc-gen-* 24 | ln -sfv "$PWD/.bin/proto-lens-protoc" "$PWD/.bin/protoc-gen-haskell" 25 | PATH="$PWD/.bin:$PATH" 26 | 27 | echo "--- buf ---" 28 | 29 | # Keep in sync with bindings/go/scip/tools.go 30 | GOBIN="$PWD/.bin" go install github.com/bufbuild/buf/cmd/buf 31 | GOBIN="$PWD/.bin" go install github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc 32 | GOBIN="$PWD/.bin" go install golang.org/x/tools/cmd/goimports 33 | GOBIN="$PWD/.bin" go install google.golang.org/protobuf/cmd/protoc-gen-go 34 | 35 | 36 | GOBIN="$PWD/.bin" ./.bin/buf generate 37 | ./.bin/goimports -w ./bindings/go/scip/scip.pb.go 38 | yarn run prettier 39 | -------------------------------------------------------------------------------- /dev/publish-release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | { 6 | if [ -z "${NEW_VERSION:-}" ]; then 7 | echo "error: Missing value for environment variable NEW_VERSION" 8 | echo "hint: Invoke this script as NEW_VERSION=M.N.P ./dev/publish-release.sh" 9 | exit 1 10 | fi 11 | 12 | if ! grep -q "## v$NEW_VERSION" CHANGELOG.md; then 13 | echo "error: Missing CHANGELOG entry for v$NEW_VERSION" 14 | echo "note: CHANGELOG entries are required for publishing releases" 15 | exit 1 16 | fi 17 | 18 | if ! grep -q "$NEW_VERSION" cmd/scip/version.txt; then 19 | echo "error: SCIP version in cmd/scip/version.txt doesn't match NEW_VERSION=$NEW_VERSION" 20 | exit 1 21 | fi 22 | 23 | if ! grep -q "version = \"$NEW_VERSION\"" bindings/rust/Cargo.toml; then 24 | echo "error: SCIP version in bindings/rust/Cargo.toml doesn't match NEW_VERSION=$NEW_VERSION" 25 | exit 1 26 | fi 27 | 28 | if ! git diff --quiet; then 29 | echo "error: Found unstaged changes; aborting." 30 | exit 1 31 | fi 32 | 33 | if ! git diff --quiet --cached; then 34 | echo "error: Found staged-but-uncommitted changes; aborting." 35 | exit 1 36 | fi 37 | 38 | if ! git rev-parse --abbrev-ref HEAD | grep -q "main"; then 39 | echo "error: Releases should be published from main but HEAD is on a different branch" >&2 40 | exit 1 41 | fi 42 | } >&2 43 | 44 | TAG="v$NEW_VERSION" 45 | git tag "$TAG" 46 | git push origin "$TAG" 47 | 48 | { 49 | echo "See the [CHANGELOG](https://github.com/sourcegraph/scip/blob/main/CHANGELOG.md) to see what's new in scip v$NEW_VERSION." 50 | echo '' 51 | echo 'Download the CLI for your current platform using:' 52 | echo '' 53 | echo '```bash' 54 | echo 'env \' 55 | echo " TAG=\"v$NEW_VERSION\" \\" 56 | echo ' OS="$(uname -s | tr '\''[:upper:]'\'' '\''[:lower:]'\'')" \' 57 | echo ' ARCH="$(uname -m | sed -e '\''s/x86_64/amd64/'\'')" \' 58 | echo ' bash -c '\''curl -L "https://github.com/sourcegraph/scip/releases/download/$TAG/scip-$OS-$ARCH.tar.gz"'\'' \' 59 | echo '| tar xzf - scip' 60 | echo '```' 61 | } | gh release create "$TAG" --title "scip v$NEW_VERSION" --notes-file - 62 | -------------------------------------------------------------------------------- /dev/sample_indexes/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore any SCIP indexes as well as any files with statistics data. 2 | *.scip 3 | *.json 4 | -------------------------------------------------------------------------------- /dev/sample_indexes/indexes-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "indexes": [ 3 | { 4 | "name": "django-1.scip", 5 | "sha256": "93fb1d73db532f1d61404220bb2d9443e42fc25f5a69246d13a13cb47feb1176" 6 | }, 7 | { 8 | "name": "flink-1.scip", 9 | "sha256": "d400dac9055bfe3260da6a9d87eab5e2e8e912424ee483baec2eecb4c11714f0" 10 | }, 11 | { 12 | "name": "kubernetes-1.scip", 13 | "sha256": "c717d08ff69470b6984dcfba12e271282bd4132f3416b9e395d9c95fd9202b87" 14 | }, 15 | { 16 | "name": "llvm-project-1.scip", 17 | "sha256": "81110140699c76cd131842ae2f6d76484ecca49786930f6efe08d75e6678b4f7" 18 | }, 19 | { 20 | "name": "materialize-1.scip", 21 | "sha256": "fac6c26746569a67ec5c9675606881f3dd0dd69c6c397893bb2ad74629a4f0f5" 22 | }, 23 | { 24 | "name": "postgres-1.scip", 25 | "sha256": "9c7ee3d7d6dc6ab9c9e2e53375ce8dd8d91c37c1f408e09d5b0fe914b1e4f384" 26 | }, 27 | { 28 | "name": "rust-1.scip", 29 | "sha256": "1b59754bd055a8022cf6c803a0fb3c2412eecb93ff5c07070f2d18994d7aecbd" 30 | }, 31 | { 32 | "name": "shopify-api-ruby-1.scip", 33 | "sha256": "a4f6543a3ab798ebd4e0a75818d3de8a818a01af68d12aa1e8d210af66736f44" 34 | }, 35 | { 36 | "name": "typescript-1.scip", 37 | "sha256": "0132ed92a82f0a41cf4378582538139917e6100f3cf5095c8a1168e634c4041f" 38 | } 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /docs/CLI.md: -------------------------------------------------------------------------------- 1 | # SCIP CLI Reference 2 | 3 | 4 | 5 | - [SCIP CLI Reference](#scip-cli-reference) 6 | - [`scip lint`](#scip-lint) 7 | - [`scip print`](#scip-print) 8 | - [`scip snapshot`](#scip-snapshot) 9 | - [`scip stats`](#scip-stats) 10 | - [`scip expt-convert`](#scip-convert) 11 | 12 | 13 | ``` 14 | NAME: 15 | scip - SCIP Code Intelligence Protocol CLI 16 | 17 | USAGE: 18 | scip [global options] command [command options] [arguments...] 19 | 20 | VERSION: 21 | v0.5.2 22 | 23 | DESCRIPTION: 24 | For more details, see the project README at: 25 | 26 | https://github.com/sourcegraph/scip 27 | 28 | COMMANDS: 29 | lint Flag potential issues with a SCIP index 30 | print Print a SCIP index for debugging 31 | snapshot Generate snapshot files for golden testing 32 | stats Output useful statistics about a SCIP index 33 | test Validate a SCIP index against test files 34 | expt-convert [EXPERIMENTAL] Convert a SCIP index to a SQLite database 35 | help, h Shows a list of commands or help for one command 36 | 37 | GLOBAL OPTIONS: 38 | --help, -h show help 39 | --version, -v print the version 40 | ``` 41 | 42 | ## `scip lint` 43 | 44 | ``` 45 | NAME: 46 | scip lint - Flag potential issues with a SCIP index 47 | 48 | USAGE: 49 | scip lint [command options] [arguments...] 50 | 51 | DESCRIPTION: 52 | Example usage: 53 | 54 | scip lint /path/to/index.scip 55 | 56 | You may want to filter the output using `grep -v ` 57 | to narrow down on certain classes of errors. 58 | ``` 59 | 60 | ## `scip print` 61 | 62 | ``` 63 | NAME: 64 | scip print - Print a SCIP index for debugging 65 | 66 | USAGE: 67 | scip print [command options] [arguments...] 68 | 69 | DESCRIPTION: 70 | WARNING: The TTY output may change over time. 71 | Do not rely on non-JSON output in scripts 72 | 73 | OPTIONS: 74 | --json Output in JSON format (default: false) 75 | --color Enable color output for TTY (no effect for JSON) (default: true) 76 | --help, -h show help 77 | ``` 78 | 79 | ## `scip snapshot` 80 | 81 | ``` 82 | NAME: 83 | scip snapshot - Generate snapshot files for golden testing 84 | 85 | USAGE: 86 | scip snapshot [command options] [arguments...] 87 | 88 | DESCRIPTION: 89 | The snapshot subcommand generates snapshot files which 90 | can be use for inspecting the output of an index in a 91 | visual way. Occurrences are marked with caret signs (^) 92 | and symbol information. 93 | 94 | For testing a SCIP indexer, you can either use this subcommand 95 | along with 'git diff' or equivalent, or you can use the dedicated 96 | 'test' subcommand for more targeted checks. 97 | 98 | OPTIONS: 99 | --from value Path to SCIP index file (default: "index.scip") 100 | --to value Path to output directory for snapshot files (default: "scip-snapshot") 101 | --project-root value Override project root in the SCIP file. For example, this can be helpful when the SCIP index was created inside a Docker image or created on another computer 102 | --strict If true, fail fast on errors (default: true) 103 | --comment-syntax value Comment syntax to use for snapshot files (default: "//") 104 | ``` 105 | 106 | ## `scip test` 107 | 108 | ``` 109 | NAME: 110 | scip test - Validate a SCIP index against test files 111 | 112 | USAGE: 113 | scip test [command options] [arguments...] 114 | 115 | DESCRIPTION: 116 | Validates whether the SCIP data as 117 | in a given SCIP index matches that specified in human-readable test files, 118 | using syntax similar to the 'snapshot subcommand'. Test file syntax reference: 119 | 120 | https://github.com/sourcegraph/scip/blob/v0.5.2/docs/test_file_format.md 121 | 122 | The test files are located based on the relative_path field 123 | in the SCIP document, interpreted relative to the the directory 124 | the CLI is invoked in. 125 | 126 | If you want to instead check all the data in a SCIP index, 127 | use the 'snapshot' subcommand. 128 | 129 | OPTIONS: 130 | --from value Path to SCIP index file (default: "index.scip") 131 | --comment-syntax value Comment syntax to use for snapshot files (default: "//") 132 | --filter value, -f value [ --filter value, -f value ] Explicit list of test files to check. Can be specified multiple times. If not specified, all files are tested. 133 | --help, -h show help 134 | ``` 135 | 136 | ## `scip stats` 137 | 138 | ``` 139 | NAME: 140 | scip stats - Output useful statistics about a SCIP index 141 | 142 | USAGE: 143 | scip stats [command options] [arguments...] 144 | 145 | OPTIONS: 146 | --from value Path to SCIP index file (default: index.scip) 147 | ``` 148 | 149 | ## `scip expt-convert` 150 | 151 | ``` 152 | NAME: 153 | scip expt-convert - [EXPERIMENTAL] Convert a SCIP index to a SQLite database 154 | 155 | USAGE: 156 | scip expt-convert [command options] [arguments...] 157 | 158 | DESCRIPTION: 159 | Converts a SCIP index to a SQLite database. 160 | 161 | For inspecting the data, use the SQLite CLI. 162 | For inspecting the schema, use .schema. 163 | 164 | Occurrences are stored opaquely as a blob to prevent the DB size from growing very quickly. 165 | 166 | OPTIONS: 167 | --output value Path to output SQLite database file (default: "index.db") 168 | --cpu-profile value Path to output prof file 169 | --help, -h show help 170 | ``` 171 | -------------------------------------------------------------------------------- /docs/scip.sprig: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # Semantic Code Intelligence Protocol (SCIP) reference 5 | 6 | {{ range .Files -}} 7 | {{ .Description }} 8 | {{ end -}} 9 | 10 | {{ range .Files -}} 11 | {{ $Enums := .Enums -}} 12 | {{ range .Messages -}} 13 | {{ $MessageName := .LongName -}} 14 | 15 | ##{{ repeat (splitList "." $MessageName | len) "#" }} {{ splitList "." $MessageName | last }} 16 | 17 | {{ .Description }} 18 | 19 | | Name | Type | Description | 20 | | ---- | ---- | ----------- | 21 | {{ range .Fields -}} 22 | | {{ .Label }} **{{ .Name }}** | {{ .Type }} | {{ splitList "\n\n" .Description | first | replace "\n" " " }} 23 | {{ end -}} 24 | 25 | {{ range .Fields -}} 26 | {{ if ne (splitList "\n\n" .Description | len) 1 }} 27 | Additional notes on **{{ .Name }}**: 28 | 29 | {{ .Description }} 30 | {{ end }} 31 | {{ end -}} 32 | 33 | {{ range $Enums -}} 34 | {{ $EnumName := .LongName -}} 35 | {{ if eq (nospace (cat $MessageName "." .Name)) $EnumName -}} 36 | 37 | ##{{ repeat (splitList "." $EnumName | len) "#" }} {{ splitList "." $EnumName | last }} 38 | 39 | {{ .Description }} 40 | 41 | | Number | Name | Description | 42 | | ------ | ---- | ----------- | 43 | {{ range .Values -}} 44 | | {{ .Number }} | {{ .Name }} | {{ splitList "\n\n" .Description | first | replace "\n" " " }} 45 | {{ end -}} 46 | 47 | {{ range .Values -}} 48 | {{ if ne (splitList "\n\n" .Description | len) 1 }} 49 | Additional notes on **{{ .Name }}**: 50 | 51 | {{ .Description }} 52 | {{ end -}} 53 | {{ end -}} 54 | 55 | {{ end -}} 56 | {{ end -}} 57 | 58 | {{ end -}} 59 | 60 | {{ range .Enums -}} 61 | {{ $EnumName := .LongName -}} 62 | {{ if eq (splitList "." $EnumName | len) 1 }} 63 | 64 | ##{{ repeat (splitList "." $EnumName | len) "#" }} {{ splitList "." $EnumName | last }} 65 | 66 | {{ .Description }} 67 | 68 | | Number | Name | Description | 69 | | ------ | ---- | ----------- | 70 | {{ range .Values -}} 71 | | {{ .Number }} | {{ .Name }} | {{ splitList "\n\n" .Description | first | replace "\n" " " }} 72 | {{ end -}} 73 | 74 | {{ range .Values -}} 75 | {{ if ne (splitList "\n\n" .Description | len) 1 }} 76 | Additional notes on **{{ .Name }}**: 77 | 78 | {{ .Description }} 79 | {{ end -}} 80 | {{ end -}} 81 | {{ end -}} 82 | {{ end -}} 83 | 84 | {{ end -}} 85 | -------------------------------------------------------------------------------- /docs/test_file_format.md: -------------------------------------------------------------------------------- 1 | # `scip test` file format 2 | 3 | The `scip test` command validates whether a provided SCIP index contains the data specified in a human-readable test file. 4 | The test file syntax is inspired by [Sublime Text's syntax highlighting tests](https://www.sublimetext.com/docs/syntax.html#testing). 5 | 6 | ## File Format 7 | 8 | Test cases are made up of a range, type, and data attribute. 9 | 10 | ### Ranges 11 | 12 | Three range selection comment formats are supported: 13 | 14 | - `// ^^^` (2 or more `^`): enforces the length of the occurrence. Will fail if the range at this location does not equal 3 characters 15 | - `// ^`: ignore length, `^` can occur at any point to any character in the occurrence 16 | - `// <-`: ignore length, and treat the character above the first comment character as the start of the occurrence, similar to Sublime Text 17 | 18 | ```js 19 | function someFunction() { 20 | // ^ ... 21 | // ^^^^^^^^^^^^ ... 22 | // <- ... 23 | } 24 | ``` 25 | 26 | ### Type and Data 27 | 28 | There are four possible types test cases. The chosen test case is determined by the first word after the range selection 29 | 30 | - `definition [symbol]` - validates that the specified range has a symbol with the role of "definition" with the specified `[symbol]` 31 | - `reference [symbol]` - validates that the specified range has a symbol with the role of "reference" with the specified `[symbol]` 32 | - `forward_definition [symbol]` - validates that the specified range has a symbol with the role of "forward_definition" with the specified `[symbol]` 33 | - `diagnostic [severity] [message]` - validates that the specified range has a diagnostic with the given `[severity]` and `[message]` 34 | 35 | ```js 36 | function someFunction() { 37 | // ^ definition scip-typescript npm test_package 1.0.0 lib/`test.js`/someFunction(). 38 | 39 | someOtherFunction() 40 | // <- reference scip-typescript npm test_package 1.0.0 lib/`test.js`/someOtherFunction(). 41 | } 42 | ``` 43 | 44 | The message for diagnostics can be specified on the following line using `>`, 45 | and may span over multiple lines. 46 | 47 | ```js 48 | function someFn() { 49 | let someVar = '' 50 | // ^ diagnostic Warning 51 | // > someVar is unused. 52 | // > remove it or use it. 53 | } 54 | ``` 55 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/sourcegraph/scip 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/bufbuild/buf v1.25.0 7 | github.com/bytedance/sonic v1.12.5 8 | github.com/cockroachdb/errors v1.8.9 9 | github.com/fatih/color v1.15.0 10 | github.com/google/go-cmp v0.5.9 11 | github.com/google/gofuzz v1.1.0 12 | github.com/hexops/autogold/v2 v2.2.1 13 | github.com/hexops/gotextdiff v1.0.3 14 | github.com/hhatto/gocloc v0.4.2 15 | github.com/jedib0t/go-pretty/v6 v6.6.3 16 | github.com/k0kubun/pp/v3 v3.1.0 17 | github.com/montanaflynn/stats v0.7.1 18 | github.com/pseudomuto/protoc-gen-doc v1.5.1 19 | github.com/smacker/go-tree-sitter v0.0.0-20220209044044-0d3022e933c3 20 | github.com/sourcegraph/beaut v0.0.0-20240611013027-627e4c25335a 21 | github.com/sourcegraph/conc v0.3.0 22 | github.com/stretchr/testify v1.8.4 23 | github.com/urfave/cli/v2 v2.25.7 24 | golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 25 | golang.org/x/tools v0.17.0 26 | google.golang.org/protobuf v1.31.0 27 | pgregory.net/rapid v1.1.0 28 | zombiezen.com/go/sqlite v1.0.0 29 | ) 30 | 31 | require ( 32 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect 33 | github.com/Masterminds/semver v1.4.2 // indirect 34 | github.com/Masterminds/sprig v2.15.0+incompatible // indirect 35 | github.com/Microsoft/go-winio v0.6.1 // indirect 36 | github.com/aokoli/goutils v1.0.1 // indirect 37 | github.com/bufbuild/connect-go v1.9.0 // indirect 38 | github.com/bufbuild/connect-opentelemetry-go v0.4.0 // indirect 39 | github.com/bufbuild/protocompile v0.5.1 // indirect 40 | github.com/bytedance/sonic/loader v0.2.0 // indirect 41 | github.com/cloudwego/base64x v0.1.4 // indirect 42 | github.com/cloudwego/iasm v0.2.0 // indirect 43 | github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect 44 | github.com/cockroachdb/redact v1.1.3 // indirect 45 | github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect 46 | github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect 47 | github.com/davecgh/go-spew v1.1.1 // indirect 48 | github.com/docker/cli v24.0.4+incompatible // indirect 49 | github.com/docker/distribution v2.8.2+incompatible // indirect 50 | github.com/docker/docker v24.0.4+incompatible // indirect 51 | github.com/docker/docker-credential-helpers v0.8.0 // indirect 52 | github.com/docker/go-connections v0.4.0 // indirect 53 | github.com/docker/go-units v0.5.0 // indirect 54 | github.com/dustin/go-humanize v1.0.1 // indirect 55 | github.com/envoyproxy/protoc-gen-validate v0.3.0-java // indirect 56 | github.com/felixge/fgprof v0.9.3 // indirect 57 | github.com/getsentry/sentry-go v0.12.0 // indirect 58 | github.com/go-chi/chi/v5 v5.0.10 // indirect 59 | github.com/go-enry/go-enry/v2 v2.7.2 // indirect 60 | github.com/go-enry/go-oniguruma v1.2.1 // indirect 61 | github.com/go-logr/logr v1.2.4 // indirect 62 | github.com/go-logr/stdr v1.2.2 // indirect 63 | github.com/gofrs/uuid/v5 v5.0.0 // indirect 64 | github.com/gogo/protobuf v1.3.2 // indirect 65 | github.com/golang/protobuf v1.5.3 // indirect 66 | github.com/google/go-containerregistry v0.15.2 // indirect 67 | github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect 68 | github.com/google/uuid v1.3.0 // indirect 69 | github.com/hexops/valast v1.4.4 // indirect 70 | github.com/huandu/xstrings v1.0.0 // indirect 71 | github.com/imdario/mergo v0.3.4 // indirect 72 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 73 | github.com/jdxcode/netrc v0.0.0-20221124155335-4616370d1a84 // indirect 74 | github.com/klauspost/compress v1.18.0 // indirect 75 | github.com/klauspost/cpuid/v2 v2.2.3 // indirect 76 | github.com/klauspost/pgzip v1.2.6 // indirect 77 | github.com/kr/pretty v0.3.1 // indirect 78 | github.com/kr/text v0.2.0 // indirect 79 | github.com/mattn/go-colorable v0.1.13 // indirect 80 | github.com/mattn/go-isatty v0.0.19 // indirect 81 | github.com/mattn/go-runewidth v0.0.15 // indirect 82 | github.com/mitchellh/go-homedir v1.1.0 // indirect 83 | github.com/moby/term v0.5.0 // indirect 84 | github.com/morikuni/aec v1.0.0 // indirect 85 | github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 // indirect 86 | github.com/ncruces/go-strftime v0.1.9 // indirect 87 | github.com/nightlyone/lockfile v1.0.0 // indirect 88 | github.com/opencontainers/go-digest v1.0.0 // indirect 89 | github.com/opencontainers/image-spec v1.1.0-rc4 // indirect 90 | github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect 91 | github.com/pkg/errors v0.9.1 // indirect 92 | github.com/pkg/profile v1.7.0 // indirect 93 | github.com/pmezard/go-difflib v1.0.0 // indirect 94 | github.com/pseudomuto/protokit v0.2.0 // indirect 95 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect 96 | github.com/rivo/uniseg v0.2.0 // indirect 97 | github.com/rogpeppe/go-internal v1.11.0 // indirect 98 | github.com/rs/cors v1.9.0 // indirect 99 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 100 | github.com/sirupsen/logrus v1.9.3 // indirect 101 | github.com/spf13/cobra v1.7.0 // indirect 102 | github.com/spf13/pflag v1.0.5 // indirect 103 | github.com/tetratelabs/wazero v1.3.0 // indirect 104 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 105 | github.com/vbatts/tar-split v0.11.3 // indirect 106 | github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect 107 | go.opentelemetry.io/otel v1.16.0 // indirect 108 | go.opentelemetry.io/otel/metric v1.16.0 // indirect 109 | go.opentelemetry.io/otel/sdk v1.16.0 // indirect 110 | go.opentelemetry.io/otel/trace v1.16.0 // indirect 111 | go.uber.org/atomic v1.11.0 // indirect 112 | go.uber.org/multierr v1.11.0 // indirect 113 | go.uber.org/zap v1.24.0 // indirect 114 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect 115 | golang.org/x/crypto v0.18.0 // indirect 116 | golang.org/x/mod v0.14.0 // indirect 117 | golang.org/x/net v0.20.0 // indirect 118 | golang.org/x/sync v0.6.0 // indirect 119 | golang.org/x/sys v0.21.0 // indirect 120 | golang.org/x/term v0.17.0 // indirect 121 | golang.org/x/text v0.14.0 // indirect 122 | google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect 123 | gopkg.in/yaml.v3 v3.0.1 // indirect 124 | modernc.org/libc v1.41.0 // indirect 125 | modernc.org/mathutil v1.6.0 // indirect 126 | modernc.org/memory v1.7.2 // indirect 127 | modernc.org/sqlite v1.27.0 // indirect 128 | mvdan.cc/gofumpt v0.5.0 // indirect 129 | ) 130 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@sourcegraph/scip-root", 3 | "private": true, 4 | "version": "0.1.0", 5 | "repository": "git@github.com:sourcegraph/scip.git", 6 | "author": "SCIP authors", 7 | "license": "MIT", 8 | "scripts": { 9 | "build": "tsc --build --force bindings/typescript", 10 | "prettier": "prettier --write --list-different '**/*.{ts,js(on)?,md,yml}'", 11 | "prettier-check": "prettier --check '**/*.{ts,js(on)?,md,yml}'" 12 | }, 13 | "workspaces": { 14 | "packages": [ 15 | "cmd/scip/tests/reprolang", 16 | "bindings/typescript" 17 | ] 18 | }, 19 | "devDependencies": { 20 | "prettier": "^3.5.3" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": ["local>sourcegraph/renovate-config"] 4 | } 5 | -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | "@types/google-protobuf@3.15.6": 6 | version "3.15.6" 7 | resolved "https://registry.yarnpkg.com/@types/google-protobuf/-/google-protobuf-3.15.6.tgz#674a69493ef2c849b95eafe69167ea59079eb504" 8 | integrity sha512-pYVNNJ+winC4aek+lZp93sIKxnXt5qMkuKmaqS3WGuTq0Bw1ZDYNBgzG5kkdtwcv+GmYJGo3yEg6z2cKKAiEdw== 9 | 10 | google-protobuf@^3.20.1: 11 | version "3.20.1" 12 | resolved "https://registry.yarnpkg.com/google-protobuf/-/google-protobuf-3.20.1.tgz#1b255c2b59bcda7c399df46c65206aa3c7a0ce8b" 13 | integrity sha512-XMf1+O32FjYIV3CYu6Tuh5PNbfNEU5Xu22X+Xkdb/DUexFlCzhvv7d5Iirm4AOwn8lv4al1YvIhzGrg2j9Zfzw== 14 | 15 | nan@^2.22.0: 16 | version "2.22.0" 17 | resolved "https://registry.yarnpkg.com/nan/-/nan-2.22.0.tgz#31bc433fc33213c97bad36404bb68063de604de3" 18 | integrity sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw== 19 | 20 | prettier@^3.5.3: 21 | version "3.5.3" 22 | resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.5.3.tgz#4fc2ce0d657e7a02e602549f053b239cb7dfe1b5" 23 | integrity sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw== 24 | 25 | protoc-gen-ts@0.8.6: 26 | version "0.8.6" 27 | resolved "https://registry.yarnpkg.com/protoc-gen-ts/-/protoc-gen-ts-0.8.6.tgz#e789a6fc3fbe09bdc119acecc349b9554ec5940e" 28 | integrity sha512-66oeorGy4QBvYjQGd/gaeOYyFqKyRmRgTpofmnw8buMG0P7A0jQjoKSvKJz5h5tNUaVkIzvGBUTRVGakrhhwpA== 29 | 30 | tree-sitter-cli@0.21.0: 31 | version "0.21.0" 32 | resolved "https://registry.yarnpkg.com/tree-sitter-cli/-/tree-sitter-cli-0.21.0.tgz#563f0a387a62c6ca57ce8dc94849e3ad7bbb0abc" 33 | integrity sha512-wA7wT5724fNQW82XDH6zT6ZcYonjrAKLCHHuhLsPcAKULrhp3rNuMvlgBdB5FUBvmjHNhtTZF/qpHenMoRJPBw== 34 | 35 | typescript@^4.9.0: 36 | version "4.9.5" 37 | resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a" 38 | integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== 39 | --------------------------------------------------------------------------------