├── .gitattributes
├── .github
├── pull_request_template.md
└── workflows
│ ├── ci.yml
│ └── docker.yml
├── .gitignore
├── .gitmodules
├── .tool-versions
├── Dockerfile
├── LICENSE
├── README.md
├── Taskfile.yml
├── cmd
└── doctree
│ ├── add.go
│ ├── index.go
│ ├── main.go
│ ├── search.go
│ ├── serve.go
│ └── util.go
├── docs
├── development.md
└── operations.md
├── doctree
├── apischema
│ └── apischema.go
├── git
│ └── git.go
├── indexer
│ ├── cli.go
│ ├── golang
│ │ └── indexer.go
│ ├── indexer.go
│ ├── javascript
│ │ └── indexer.go
│ ├── markdown
│ │ ├── indexer.go
│ │ └── indexer_test.go
│ ├── python
│ │ └── indexer.go
│ ├── search.go
│ └── zig
│ │ └── indexer.go
├── schema
│ └── schema.go
└── sourcegraph
│ ├── client.go
│ ├── graphql.go
│ ├── query_def_ref_impl.go
│ └── types.go
├── frontend
├── assets.go
├── elm.json
├── package-lock.json
├── package.json
├── public
│ ├── android-chrome-192x192.png
│ ├── android-chrome-512x512.png
│ ├── apple-touch-icon.png
│ ├── browserconfig.xml
│ ├── favicon-16x16.png
│ ├── favicon-32x32.png
│ ├── favicon.ico
│ ├── index-cloud.html
│ ├── index.html
│ ├── mascot.svg
│ ├── mstile-150x150.png
│ ├── opendata.js
│ ├── safari-pinned-tab.svg
│ ├── site.webmanifest
│ └── stylesheet.css
└── src
│ ├── API.elm
│ ├── APISchema.elm
│ ├── Flags.elm
│ ├── Home.elm
│ ├── Main.elm
│ ├── Markdown.elm
│ ├── Ports.elm
│ ├── Project.elm
│ ├── Route.elm
│ ├── Schema.elm
│ ├── Search.elm
│ ├── Style.elm
│ └── Util.elm
├── go.mod
├── go.sum
└── renovate.json
/.gitattributes:
--------------------------------------------------------------------------------
1 | * text=auto eol=lf
2 |
3 |
4 |
--------------------------------------------------------------------------------
/.github/pull_request_template.md:
--------------------------------------------------------------------------------
1 |
2 | - [ ] By selecting this checkbox, I agree to license my contributions to this project under the license(s) described in the LICENSE file, and I have the right to do so or have received
3 | permission to do so by an employer or client I am producing work for whom has this right.
4 |
--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
1 | name: CI
2 | on:
3 | - push
4 | - pull_request
5 | jobs:
6 | x86_64-macos:
7 | runs-on: macos-latest
8 | # We want to run on external PRs, but not on our own internal PRs as they'll be run by the push
9 | # to the branch.
10 | if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
11 | permissions:
12 | contents: write # for release creation
13 | steps:
14 | - name: Checkout
15 | uses: actions/checkout@v3
16 | - name: Install Zig
17 | run: |
18 | brew install xz
19 | sudo sh -c 'wget -c https://ziglang.org/builds/zig-macos-x86_64-0.10.0-dev.2439+c84f5a5f9.tar.xz -O - | tar -xJ --strip-components=1 -C /usr/local/bin'
20 | - name: Install dependencies
21 | run: |
22 | brew install go-task/tap/go-task
23 | brew uninstall go@1.17
24 | brew install go@1.18
25 | task setup
26 | - name: Run tests
27 | run: task test
28 | - name: Cross-compile for every OS
29 | run: task cross-compile
30 | - name: Record latest release version
31 | id: recorded_release_version
32 | run: echo "::set-output name=commit::$(git log --oneline | head -n1 | cut -d " " -f1)"
33 | - name: Release
34 | if: success() && github.ref == 'refs/heads/main' && github.event_Name == 'push' && github.repository == 'sourcegraph/doctree'
35 | run: |
36 | export RELEASE_NAME="$(date +%Y-%m-%d)-$RELEASE_COMMIT"
37 | gh release create "release-$RELEASE_NAME" --title "Release of main @ $RELEASE_COMMIT"
38 | gh release upload "release-$RELEASE_NAME" out/doctree-aarch64-macos
39 | gh release upload "release-$RELEASE_NAME" out/doctree-x86_64-linux
40 | gh release upload "release-$RELEASE_NAME" out/doctree-x86_64-macos
41 | gh release upload "release-$RELEASE_NAME" out/doctree-x86_64-windows.exe
42 | env:
43 | RELEASE_COMMIT: ${{steps.recorded_release_version.outputs.commit}}
44 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
45 |
--------------------------------------------------------------------------------
/.github/workflows/docker.yml:
--------------------------------------------------------------------------------
1 | name: Docker images
2 | on:
3 | workflow_run:
4 | workflows: ["CI"]
5 | types:
6 | - completed
7 | jobs:
8 | main:
9 | if: ${{ github.event.workflow_run.conclusion == 'success' }}
10 | runs-on: ubuntu-latest
11 | steps:
12 | - uses: actions/checkout@v3
13 | - name: Install Go
14 | uses: actions/setup-go@v3
15 | with:
16 | go-version: "1.18"
17 | - name: Install Zig
18 | run: |
19 | sudo apt install xz-utils
20 | sudo sh -c 'wget -c https://ziglang.org/builds/zig-linux-x86_64-0.10.0-dev.2439+c84f5a5f9.tar.xz -O - | tar -xJ --strip-components=1 -C /usr/local/bin'
21 | - name: Install Task
22 | uses: arduino/setup-task@v1
23 | - name: Build Docker image
24 | run: |
25 | task setup
26 | task build-image
27 | - name: Record latest release version
28 | id: recorded_release_version
29 | run: echo "::set-output name=commit::$(git log --oneline | head -n1 | cut -d " " -f1)"
30 | - name: Publish Docker image
31 | if: success() && github.ref == 'refs/heads/main' && github.repository == 'sourcegraph/doctree'
32 | run: |
33 | export RELEASE_NAME="$(date --rfc-3339=date)-$RELEASE_COMMIT"
34 | echo "$DOCKER_PASSWORD" | docker login -u="$DOCKER_USERNAME" --password-stdin
35 | docker tag "sourcegraph/doctree:dev" "sourcegraph/doctree:$RELEASE_NAME"
36 | docker push "sourcegraph/doctree:$RELEASE_NAME"
37 | docker tag "sourcegraph/doctree:dev" "sourcegraph/doctree:latest"
38 | docker push "sourcegraph/doctree:latest"
39 | env:
40 | RELEASE_COMMIT: ${{steps.recorded_release_version.outputs.commit}}
41 | DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
42 | DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
43 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | elm-stuff
3 | node_modules
4 | dist
5 | *~
6 | .vscode/
7 | generated/
8 | .bin
9 | .task
10 | out/
11 |
--------------------------------------------------------------------------------
/.gitmodules:
--------------------------------------------------------------------------------
1 | [submodule "libs/sinter"]
2 | path = libs/sinter
3 | url = https://github.com/hexops/sinter
4 |
--------------------------------------------------------------------------------
/.tool-versions:
--------------------------------------------------------------------------------
1 | golang 1.18
2 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 |
2 | ###############################
3 | # Build the doctree Go binary #
4 | ###############################
5 | FROM golang:1.18-alpine as builder
6 |
7 | RUN apk add --no-cache git build-base
8 | COPY . /doctree
9 | ENV GOBIN /out
10 | RUN cd /doctree && go install ./cmd/doctree
11 |
12 | # Dockerfile based on guidelines at https://github.com/hexops/dockerfile
13 | FROM alpine:3.16@sha256:686d8c9dfa6f3ccfc8230bc3178d23f84eeaf7e457f36f271ab1acc53015037c
14 |
15 | # Non-root user for security purposes.
16 | #
17 | # UIDs below 10,000 are a security risk, as a container breakout could result
18 | # in the container being ran as a more privileged user on the host kernel with
19 | # the same UID.
20 | #
21 | # Static GID/UID is also useful for chown'ing files outside the container where
22 | # such a user does not exist.
23 | RUN addgroup --gid 10001 --system nonroot \
24 | && adduser --uid 10000 --system --ingroup nonroot --home /home/nonroot nonroot
25 |
26 | # Copy Go binary from builder image
27 | COPY --from=builder /out/ /usr/local/bin
28 |
29 | # For doctree to inspect Git repository URIs.
30 | RUN apk add --no-cache git
31 |
32 | # TODO: Although libsinter is built with musl, CGO uses glibc still. Should remove that dependency.
33 | RUN apk add --no-cache libgcc libstdc++
34 |
35 | # Create data volume.
36 | RUN mkdir -p /home/nonroot/.doctree
37 | RUN chown -R nonroot:nonroot /home/nonroot/.doctree
38 | VOLUME /home/nonroot/.doctree
39 |
40 | # Tini allows us to avoid several Docker edge cases, see https://github.com/krallin/tini.
41 | # NOTE: See https://github.com/hexops/dockerfile#is-tini-still-required-in-2020-i-thought-docker-added-it-natively
42 | RUN apk add --no-cache tini
43 | ENTRYPOINT ["/sbin/tini", "--", "doctree"]
44 |
45 | # bind-tools is needed for DNS resolution to work in *some* Docker networks, but not all.
46 | # This applies to nslookup, Go binaries, etc. If you want your Docker image to work even
47 | # in more obscure Docker environments, use this.
48 | RUN apk add --no-cache bind-tools
49 |
50 | # Use the non-root user to run our application
51 | USER nonroot
52 |
53 | # Default arguments for your app (remove if you have none):
54 | EXPOSE 3333
55 | CMD ["serve"]
56 |
57 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright 2018 Sourcegraph, Inc.
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # doctree: 100% open-source library docs tool for every language
2 |
3 | doctree provides first-class library documentation for every language (based on tree-sitter), with symbol search & more. If connected to Sourcegraph, it can automatically surface real-world usage examples.
4 |
5 | ## Try it at [doctree.org](https://doctree.org)
6 |
7 | [](https://doctree.org)
8 |
9 | ## Run locally, self-host, or use doctree.org
10 |
11 | doctree is a single binary, lightweight, and designed to run on your local machine. It can be self-hosted, and used via doctree.org with any GitHub repository.
12 |
13 | ## Experimental! Early stages!
14 |
15 | Extremely early stages, we're working on adding more languages, polishing the experience, and adding usage examples. It's all very early and not yet ready for production use, please bear with us!
16 |
17 | Please see [the v1.0 roadmap](https://github.com/sourcegraph/doctree/issues/27) for more, ideas welcome!
18 |
19 | ## Join us on Discord
20 |
21 | If you think what we're building is a good idea, we'd love to hear your thoughts!
22 | [Discord invite](https://discord.gg/vqsBW8m5Y8)
23 |
24 | ## Language support
25 |
26 | Adding support for more languages is easy. To request support for a language [comment on this issue](https://github.com/sourcegraph/doctree/issues/10)
27 |
28 | | language | functions | types | methods | consts/vars | search | usage examples | code intel |
29 | |----------|-----------|-------|---------|-------------|--------|----------------|------------|
30 | | Go | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ |
31 | | Python | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ |
32 | | Zig | ✅ | ❌ | partial | ❌ | ✅ | ❌ | ❌ |
33 | | Markdown | n/a | ❌ | n/a | n/a | ✅ | n/a | n/a |
34 |
35 | ## Installation
36 |
37 |
38 | macOS (Apple Silicon)
39 |
40 | ```sh
41 | curl -L https://github.com/sourcegraph/doctree/releases/latest/download/doctree-aarch64-macos -o /usr/local/bin/doctree
42 | chmod +x /usr/local/bin/doctree
43 | ```
44 |
45 |
46 |
47 |
48 | macOS (Intel)
49 |
50 | ```sh
51 | curl -L https://github.com/sourcegraph/doctree/releases/latest/download/doctree-x86_64-macos -o /usr/local/bin/doctree
52 | chmod +x /usr/local/bin/doctree
53 | ```
54 |
55 |
56 |
57 |
58 | Linux (x86_64)
59 |
60 | ```sh
61 | curl -L https://github.com/sourcegraph/doctree/releases/latest/download/doctree-x86_64-linux -o /usr/local/bin/doctree
62 | chmod +x /usr/local/bin/doctree
63 | ```
64 |
65 |
66 |
67 |
68 | Windows (x86_64)
69 | In an administrator PowerShell, run:
70 |
71 | ```powershell
72 | New-Item -ItemType Directory 'C:\Program Files\Sourcegraph'
73 |
74 | Invoke-WebRequest https://github.com/sourcegraph/doctree/releases/latest/download/doctree-x86_64-windows.exe -OutFile 'C:\Program Files\Sourcegraph\doctree.exe'
75 |
76 | [Environment]::SetEnvironmentVariable('Path', [Environment]::GetEnvironmentVariable('Path', [EnvironmentVariableTarget]::Machine) + ';C:\Program Files\Sourcegraph', [EnvironmentVariableTarget]::Machine)
77 | $env:Path += ';C:\Program Files\Sourcegraph'
78 | ```
79 |
80 | Or download [the exe file](https://github.com/sourcegraph/doctree/releases/latest/download/doctree-x86_64-windows.exe) and install it wherever you like.
81 |
82 |
83 |
84 |
85 | Via Docker
86 |
87 | ```sh
88 | docker run -it --publish 3333:3333 --rm --name doctree --volume ~/.doctree:/home/nonroot/.doctree sourcegraph/doctree:latest
89 | ```
90 |
91 | In a folder with Go code you'd like to see docs for, index it (for a large project like `golang/go` expect it to take ~52s for now. It's not multi-threaded.):
92 |
93 | ```sh
94 | docker run -it --volume $(pwd):/index --volume ~/.doctree:/home/nonroot/.doctree --entrypoint=sh sourcegraph/doctree:latest -c "cd /index && doctree index ."
95 | ```
96 |
97 |
98 |
99 |
100 | DigitalOcean user data
101 |
102 | ```sh
103 | #!/bin/bash
104 |
105 | apt update -y && apt upgrade -y && apt install -y docker.io
106 | apt install -y git
107 |
108 | mkdir -p $HOME/.doctree && chown 10000:10001 -R $HOME/.doctree
109 |
110 | # Index golang/go repository
111 | git clone https://github.com/golang/go
112 | chown 10000:10001 -R go
113 | cd go
114 | docker run -i --volume $(pwd):/index --volume $HOME/.doctree:/home/nonroot/.doctree --entrypoint=sh sourcegraph/doctree:latest -c "cd /index && doctree index ."
115 |
116 | # Run server
117 | docker rm -f doctree || true
118 | docker run -d --rm --name doctree -p 80:3333 --volume $HOME/.doctree:/home/nonroot/.doctree sourcegraph/doctree:latest
119 | ```
120 |
121 |
122 |
123 | ## Usage
124 |
125 | Run the server:
126 |
127 | ```sh
128 | doctree serve
129 | ```
130 |
131 | Index a Go project (takes ~52s for a large project like golang/go itself, will be improved soon):
132 |
133 | ```sh
134 | doctree index .
135 | ```
136 |
137 | Navigate to http://localhost:3333
138 |
139 | ## Contributing
140 |
141 | We'd love any contributions!
142 |
143 | To get started see [docs/development.md](docs/development.md) and the [language support tracking issue](https://github.com/sourcegraph/doctree/issues/10).
144 |
145 | ## Changelog
146 |
147 | ### v0.2 (not yet released)
148 |
149 | * Page downloads are now even slimmer (a few KiB for a a large Go package page.)
150 | * Fixed an issue where root Go project pages (e.g. `github.com/gorilla/mux` which contains only one Go package) would not render.
151 |
152 | ### v0.1
153 |
154 | * Go, Python, Zig, and Markdown basic support
155 | * Basic search navigation experience based on [experimental Sinter search filters](https://github.com/hexops/sinter/blob/c87e502f3cfd468d3d1263b7caf7cea94ff6d084/src/filter.zig#L18-L85)
156 | * Searching globally across all projects, and within specific projects is now possible.
157 | * Searching within a specific language is now supported (add "go", "python", "md" / "markdown" to front of your query string.)
158 | * Markdown files now have headers and sub-headers indexed for search (e.g. `# About doctree > Installation` shows up in search)
159 | * Basic Markdown frontmatter support.
160 | * Initial [doctree schema format](https://github.com/sourcegraph/doctree/blob/main/doctree/schema/schema.go)
161 | * Experimental (not yet ready for use) auto-indexing, `doctree add .` to monitor a project for file changes and re-index automatically.
162 | * Docker images, single-binary downloads for every OS cross-compiled via Zig compiler.
163 | * Initial [v1.0 roadmap](https://github.com/sourcegraph/doctree/issues/27), [language support tracking issue](https://github.com/sourcegraph/doctree/issues/10)
164 |
165 | Special thanks: [@KShivendu](https://github.com/KShivendu) (Python support), [@slimsag](https://github.com/slimsag) (Zig support)
166 |
--------------------------------------------------------------------------------
/Taskfile.yml:
--------------------------------------------------------------------------------
1 | version: "3"
2 |
3 | tasks:
4 | default:
5 | desc: Run development server, watching code files for changes.
6 | cmds:
7 | - task --parallel fmt-lint backend frontend --watch
8 |
9 | cloud:
10 | desc: Run development server, watching code files for changes.
11 | cmds:
12 | - task --parallel fmt-lint backend-cloud frontend --watch
13 |
14 | fmt-lint:
15 | desc: "Run formatters and linters"
16 | cmds:
17 | - task: fmt
18 | - task: lint
19 |
20 | backend:
21 | desc: Run "doctree serve" backend
22 | cmds:
23 | - .bin/doctree serve
24 | env:
25 | ELM_DEBUG_SERVER: "http://localhost:8000/"
26 | deps:
27 | - build-go-debug
28 |
29 | backend-cloud:
30 | desc: Run "doctree serve -cloud" backend
31 | cmds:
32 | - .bin/doctree serve -cloud
33 | env:
34 | ELM_DEBUG_SERVER: "http://localhost:8000/"
35 | deps:
36 | - build-go-debug
37 |
38 | frontend:
39 | desc: Run Elm frontend dev server
40 | dir: frontend
41 | cmds:
42 | - npx elm-live --dir public --pushstate true src/Main.elm -- --output=public/dist/elm.js
43 |
44 | build:
45 | desc: Build Go + Elm code in release mode
46 | cmds:
47 | - task: build-go-release
48 | deps:
49 | - task: build-elm-release
50 |
51 | build-elm-debug:
52 | desc: Build Elm frontend code (debug mode)
53 | dir: frontend
54 | cmds:
55 | - mkdir -p public/dist/
56 | - npx elm make --debug src/Main.elm --output ./public/dist/elm.js
57 | sources:
58 | - ./frontend/src/**/*.elm
59 | generates:
60 | - ./frontend/public/dist/elm.js
61 |
62 | build-elm-release:
63 | desc: Build Elm frontend code (release mode, minified, etc.)
64 | dir: frontend
65 | cmds:
66 | - mkdir -p public/dist/
67 | - npx elm make src/Main.elm --optimize --output ./public/dist/elm.js
68 | sources:
69 | - ./frontend/src/**/*.elm
70 | generates:
71 | - ./frontend/public/dist/elm.js
72 |
73 | build-go-debug:
74 | desc: Build .bin/doctree (debug)
75 | cmds:
76 | - mkdir -p .bin
77 | - go build -ldflags "-w" -o .bin/doctree ./cmd/doctree
78 | sources:
79 | - ./**/*.go
80 | - libs/sinter/**/*.zig
81 | generates:
82 | - .bin/doctree
83 | deps:
84 | - task: build-zig-debug
85 |
86 | build-go-release:
87 | desc: Build .bin/doctree (release)
88 | cmds:
89 | - mkdir -p .bin
90 | - go build -o .bin/doctree ./cmd/doctree
91 | sources:
92 | - ./**/*.go
93 | - ./frontend/public/**
94 | - libs/sinter/**/*.zig
95 | generates:
96 | - .bin/doctree
97 | deps:
98 | - task: build-zig-release
99 |
100 | build-zig-debug:
101 | desc: Build Zig code in debug mode
102 | cmds:
103 | - rm -f libs/sinter/bindings/sinter-go/go.mod
104 | - cd libs/sinter && zig build
105 | sources:
106 | - libs/sinter/**/*.zig
107 |
108 | build-zig-release:
109 | desc: Build Zig code in release mode
110 | cmds:
111 | - rm -f libs/sinter/bindings/sinter-go/go.mod
112 | - cd libs/sinter && zig build -Drelease-fast=true
113 | sources:
114 | - libs/sinter/**/*.zig
115 |
116 | test:
117 | desc: Run all tests
118 | cmds:
119 | - go test ./...
120 | deps:
121 | - task: build-zig-debug
122 |
123 | test-race:
124 | desc: Run all tests (checking for race conditions, slow)
125 | cmds:
126 | - go test -race ./...
127 | deps:
128 | - task: build-zig-debug
129 |
130 | generate:
131 | desc: Produce generated code
132 | cmds:
133 | - go generate ./...
134 | sources:
135 | - ./**/*.go
136 |
137 | lint:
138 | desc: Lint code
139 | cmds:
140 | # Using --go=1.17 for now because of https://github.com/golangci/golangci-lint/issues/2649
141 | - .bin/golangci-lint run --go=1.17 ./...
142 | sources:
143 | - ./**/*.go
144 | deps:
145 | - build-tools
146 |
147 | fmt:
148 | desc: Format code
149 | cmds:
150 | - .bin/gofumpt -l -w .
151 | sources:
152 | - ./**/*.go
153 | deps:
154 | - build-tools
155 |
156 | setup:
157 | desc: Install dependencies, pull submodules
158 | cmds:
159 | - git submodule update --init --recursive
160 | - cd frontend && npm install
161 |
162 | build-tools:
163 | desc: Build tool dependencies (golangci-lint, etc.)
164 | cmds:
165 | - GOBIN="$(pwd)/.bin" go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.45.2
166 | - GOBIN="$(pwd)/.bin" go install mvdan.cc/gofumpt@latest
167 | status:
168 | - test -f .bin/golangci-lint
169 | - test -f .bin/gofumpt
170 |
171 | build-image:
172 | desc: Build sourcegraph/doctree:dev Docker image
173 | cmds:
174 | - docker build --no-cache -t sourcegraph/doctree:dev .
175 | deps:
176 | - task: cc-x86_64-linux
177 | - task: build-elm-release
178 |
179 | run-image:
180 | desc: Run sourcegraph/doctree:dev Docker image
181 | cmds:
182 | - docker run -it sourcegraph/doctree:dev
183 |
184 | cross-compile:
185 | desc: "Cross compile binaries using Zig (requires Go 1.18+, Zig 0.10+, macOS Monterey 12+ host machine)"
186 | cmds:
187 | - task: cc-x86_64-linux
188 | - task: cc-x86_64-macos
189 | - task: cc-aarch64-macos
190 | - task: cc-x86_64-windows
191 |
192 | cc-x86_64-linux:
193 | desc: "Cross compile to x86_64-linux using Zig"
194 | cmds:
195 | - rm -f libs/sinter/bindings/sinter-go/go.mod
196 | - cd libs/sinter && zig build -Drelease-fast=true -Dtarget=x86_64-linux-musl
197 | - go build -o out/doctree-x86_64-linux ./cmd/doctree/
198 | env:
199 | CGO_ENABLED: "1"
200 | GOOS: "linux"
201 | GOARCH: "amd64"
202 | CC: "zig cc -target x86_64-linux-musl"
203 | CXX: "zig c++ -target x86_64-linux-musl"
204 | sources:
205 | - ./**/*.go
206 | - ./frontend/public/**
207 | generates:
208 | - out/doctree-x86_64-macos
209 | deps:
210 | - task: build-elm-release
211 |
212 | cc-x86_64-macos:
213 | desc: "Cross compile to x86_64-macos using Zig (REQUIRES XCODE, ONLY WORKS ON MACOS)"
214 | cmds:
215 | - rm -f libs/sinter/bindings/sinter-go/go.mod
216 | - cd libs/sinter && zig build -Drelease-fast=true -Dtarget=x86_64-macos
217 | - go build -o out/doctree-x86_64-macos -buildmode=pie -ldflags "-s -w -linkmode external" ./cmd/doctree/
218 | env:
219 | CGO_ENABLED: "1"
220 | GOOS: "darwin"
221 | GOARCH: "amd64"
222 | CC: "zig cc -target x86_64-macos -F /System/Library/Frameworks --sysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"
223 | CXX: "zig c++ -target x86_64-macos -F /System/Library/Frameworks --sysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"
224 | sources:
225 | - ./**/*.go
226 | - ./frontend/public/**
227 | generates:
228 | - out/doctree-x86_64-macos
229 | deps:
230 | - task: build-elm-release
231 |
232 | cc-aarch64-macos:
233 | desc: "Cross compile to aarch64-macos using Zig (REQUIRES XCODE, ONLY WORKS ON MACOS 12+)"
234 | cmds:
235 | - rm -f libs/sinter/bindings/sinter-go/go.mod
236 | - cd libs/sinter && zig build -Drelease-fast=true -Dtarget=aarch64-macos
237 | - go build -o out/doctree-aarch64-macos -buildmode=pie -ldflags "-s -w -linkmode external" ./cmd/doctree/
238 | env:
239 | CGO_ENABLED: "1"
240 | GOOS: "darwin"
241 | GOARCH: "arm64"
242 | CC: "zig cc -target aarch64-macos -F /System/Library/Frameworks --sysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"
243 | CXX: "zig c++ -target aarch64-macos -F /System/Library/Frameworks --sysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"
244 | sources:
245 | - ./**/*.go
246 | - ./frontend/public/**
247 | generates:
248 | - out/doctree-aarch64-macos
249 | deps:
250 | - task: build-elm-release
251 |
252 | cc-x86_64-windows:
253 | desc: "Cross compile to x86_64-windows using Zig"
254 | cmds:
255 | - rm -f libs/sinter/bindings/sinter-go/go.mod
256 | - cd libs/sinter && zig build -Drelease-fast=true -Dtarget=x86_64-windows-gnu
257 | - go build -o out/doctree-x86_64-windows.exe ./cmd/doctree/
258 | env:
259 | CGO_ENABLED: "1"
260 | GOOS: "windows"
261 | GOARCH: "amd64"
262 | CC: "zig cc -Wno-dll-attribute-on-redeclaration -target x86_64-windows-gnu"
263 | CXX: "zig c++ -target x86_64-windows-gnu"
264 | sources:
265 | - ./**/*.go
266 | - ./frontend/public/**
267 | generates:
268 | - out/doctree-x86_64-windows.exe
269 | deps:
270 | - task: build-elm-release
271 |
272 | ops-ssh:
273 | desc: "SSH doctree.org"
274 | cmds:
275 | - ssh root@164.92.76.86
276 |
277 | ops-add-repo:
278 | desc: "Add a repo to doctree.org (requires operations access)"
279 | cmds:
280 | - ssh $HOST -t 'rm -rf repo/'
281 | - ssh $HOST -t 'git clone --depth 1 {{.CLI_ARGS}} repo'
282 | - ssh $HOST -t 'chown 10000:10001 -R repo'
283 | - ssh $HOST -t 'cd repo/ && docker run -i --volume $(pwd):/index --volume /.doctree:/home/nonroot/.doctree --entrypoint=sh sourcegraph/doctree:latest -c "cd /index && doctree index ."'
284 | # TODO why is this necessary? Docker volume becomes not in sync with host, how come?
285 | - ssh $HOST -t 'docker restart doctree'
286 | env:
287 | HOST: root@164.92.76.86
288 |
289 | ops-add-repo-zigstd:
290 | desc: "Add a ziglang/zig stdlib to doctree.org (requires operations access)"
291 | cmds:
292 | # TODO: Right now, we must cd into lib/std because otherwise indexing ziglang/zig fails!
293 | - ssh $HOST -t 'rm -rf repo/'
294 | - ssh $HOST -t 'git clone --depth 1 {{.CLI_ARGS}} repo'
295 | - ssh $HOST -t 'chown 10000:10001 -R repo'
296 | - ssh $HOST -t 'cd repo/lib/std && docker run -i --volume $(pwd):/index --volume /.doctree:/home/nonroot/.doctree --entrypoint=sh sourcegraph/doctree:latest -c "cd /index && doctree index -project=github.com/ziglang/zig ."'
297 | # TODO why is this necessary? Docker volume becomes not in sync with host, how come?
298 | - ssh $HOST -t 'docker restart doctree'
299 | env:
300 | HOST: root@164.92.76.86
301 |
302 | ops-remove-all:
303 | desc: "Remove all data from doctree.org (requires operations access)"
304 | cmds:
305 | - ssh $HOST -t 'rm -rf /.doctree'
306 | - ssh $HOST -t 'mkdir /.doctree'
307 | - ssh $HOST -t 'chown 10000:10001 -R /.doctree'
308 | env:
309 | HOST: root@164.92.76.86
310 |
311 | ops-add-sample-repos:
312 | desc: "Add sample repos to doctree.org (requires operations access)"
313 | cmds:
314 | - task ops-add-repo -- github.com/Sobeston/ziglearn
315 | - task ops-add-repo -- github.com/alanhamlett/pip-update-requirements
316 | - task ops-add-repo -- github.com/caddyserver/caddy
317 | - task ops-add-repo -- github.com/cozy/cozy-stack
318 | - task ops-add-repo -- github.com/django/django
319 | - task ops-add-repo -- github.com/elastic/elasticsearch
320 | - task ops-add-repo -- github.com/facebook/react
321 | - task ops-add-repo -- github.com/gogs/gogs
322 | - task ops-add-repo -- github.com/golang/go
323 | - task ops-add-repo -- github.com/gorilla/mux
324 | - task ops-add-repo -- github.com/hexops/mach
325 | - task ops-add-repo -- github.com/jertel/elastalert2
326 | - task ops-add-repo -- github.com/kevinwojo/mvs
327 | - task ops-add-repo -- github.com/kooparse/zalgebra
328 | - task ops-add-repo -- github.com/kooparse/zgltf
329 | - task ops-add-repo -- github.com/paulshen/css-editor
330 | - task ops-add-repo -- github.com/pytest-dev/pytest
331 | - task ops-add-repo -- github.com/rasahq/rasa
332 | - task ops-add-repo -- github.com/renode/renode
333 | - task ops-add-repo -- github.com/saltbo/zpan
334 | - task ops-add-repo -- github.com/smallstep/certificates
335 | - task ops-add-repo -- github.com/sourcegraph/sourcegraph
336 | - task ops-add-repo -- github.com/wakatime/wakatime-cli
337 | - task ops-add-repo -- github.com/zephyrproject-rtos/zephyr
338 | - task ops-add-repo -- github.com/ziglang/zig
339 | env:
340 | HOST: root@164.92.76.86
341 |
342 | ops-deploy:
343 | desc: "Deploy latest version to doctree.org (requires operations access)"
344 | cmds:
345 | - ssh $HOST -t 'docker pull sourcegraph/doctree:latest'
346 | - ssh $HOST -t 'docker rm -f doctree || true'
347 | - ssh $HOST -t 'docker run -d --restart=always --name doctree -p 80:3333 --volume /.doctree:/home/nonroot/.doctree sourcegraph/doctree:latest serve -cloud'
348 | - ssh $HOST -t 'docker ps'
349 | env:
350 | HOST: root@164.92.76.86
351 |
352 | dev-clone-sample-repos:
353 | desc: "Clone sample repos for dev environment"
354 | cmds:
355 | - rm -rf ../doctree-samples/ && mkdir ../doctree-samples/
356 | - cd ../doctree-samples/ && git clone --depth 1 https://github.com/golang/go
357 | - cd ../doctree-samples/ && git clone --depth 1 https://github.com/gorilla/mux
358 | - cd ../doctree-samples/ && git clone --depth 1 https://github.com/django/django
359 | - cd ../doctree-samples/ && git clone --depth 1 https://github.com/Sobeston/ziglearn
360 | - cd ../doctree-samples/ && git clone --depth 1 https://github.com/ziglang/zig
361 | - cd ../doctree-samples/ && git clone --depth 1 https://github.com/tailwindlabs/tailwindcss
362 |
363 | dev-index-sample-repos:
364 | desc: "Index sample repos for dev environment"
365 | cmds:
366 | # Go
367 | - cd ../doctree-samples/go && ../../doctree/.bin/doctree index .
368 | - cd ../doctree-samples/mux && ../../doctree/.bin/doctree index .
369 | # Python
370 | - cd ../doctree-samples/django && ../../doctree/.bin/doctree index .
371 | # Markdown
372 | - cd ../doctree-samples/ziglearn && ../../doctree/.bin/doctree index .
373 | # Zig
374 | - cd ../doctree-samples/ziglang/lib/std && ../../doctree/.bin/doctree index -project=github.com/ziglang/zig .
375 | # JavaScript
376 | - cd ../doctree-samples/tailwindcss && ../../doctree/.bin/doctree index .
377 |
--------------------------------------------------------------------------------
/cmd/doctree/add.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "flag"
6 | "fmt"
7 | "path/filepath"
8 |
9 | "github.com/hexops/cmder"
10 | "github.com/pkg/errors"
11 | "github.com/sourcegraph/doctree/doctree/indexer"
12 | )
13 |
14 | func init() {
15 | const usage = `
16 | Examples:
17 |
18 | Register current directory for auto-indexing:
19 |
20 | $ doctree add .
21 | `
22 | // Parse flags for our subcommand.
23 | flagSet := flag.NewFlagSet("add", flag.ExitOnError)
24 | dataDirFlag := flagSet.String("data-dir", defaultDataDir(), "where doctree stores its data")
25 | projectFlag := flagSet.String("project", defaultProjectName("."), "name of the project")
26 |
27 | // Handles calls to our subcommand.
28 | handler := func(args []string) error {
29 | _ = flagSet.Parse(args)
30 | if flagSet.NArg() != 1 {
31 | return &cmder.UsageError{}
32 | }
33 | dir := flagSet.Arg(0)
34 | if dir != "." {
35 | *projectFlag = defaultProjectName(dir)
36 | }
37 |
38 | projectPath, err := filepath.Abs(dir)
39 | if err != nil {
40 | return errors.Wrap(err, "projectPath")
41 | }
42 | autoIndexPath := filepath.Join(*dataDirFlag, "autoindex")
43 |
44 | // Read JSON from ~/.doctree/autoindex
45 | autoIndexedProjects, err := indexer.ReadAutoIndex(autoIndexPath)
46 | if err != nil {
47 | return err
48 | }
49 |
50 | // Update the autoIndexProjects array
51 | autoIndexedProjects[projectPath] = indexer.AutoIndexedProject{
52 | Name: *projectFlag,
53 | }
54 |
55 | err = indexer.WriteAutoIndex(autoIndexPath, autoIndexedProjects)
56 | if err != nil {
57 | return err
58 | }
59 |
60 | // Run indexers on the newly registered dir
61 | ctx := context.Background()
62 | return indexer.RunIndexers(ctx, projectPath, *dataDirFlag, *projectFlag)
63 | }
64 |
65 | // Register the command.
66 | commands = append(commands, &cmder.Command{
67 | FlagSet: flagSet,
68 | Aliases: []string{},
69 | Handler: handler,
70 | UsageFunc: func() {
71 | fmt.Fprintf(flag.CommandLine.Output(), "Usage of 'doctree %s':\n", flagSet.Name())
72 | flagSet.PrintDefaults()
73 | fmt.Fprintf(flag.CommandLine.Output(), "%s", usage)
74 | },
75 | })
76 | }
77 |
--------------------------------------------------------------------------------
/cmd/doctree/index.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "flag"
6 | "fmt"
7 |
8 | "github.com/hexops/cmder"
9 |
10 | "github.com/sourcegraph/doctree/doctree/indexer"
11 | )
12 |
13 | func init() {
14 | const usage = `
15 | Examples:
16 |
17 | Index all code in the current directory:
18 |
19 | $ doctree index .
20 |
21 | `
22 |
23 | // Parse flags for our subcommand.
24 | flagSet := flag.NewFlagSet("index", flag.ExitOnError)
25 | dataDirFlag := flagSet.String("data-dir", defaultDataDir(), "where doctree stores its data")
26 | projectFlag := flagSet.String("project", defaultProjectName("."), "name of the project")
27 |
28 | // Handles calls to our subcommand.
29 | handler := func(args []string) error {
30 | _ = flagSet.Parse(args)
31 | if flagSet.NArg() != 1 {
32 | return &cmder.UsageError{}
33 | }
34 | dir := flagSet.Arg(0)
35 | if dir != "." {
36 | *projectFlag = defaultProjectName(dir)
37 | }
38 |
39 | ctx := context.Background()
40 | return indexer.RunIndexers(ctx, dir, *dataDirFlag, *projectFlag)
41 | }
42 |
43 | // Register the command.
44 | commands = append(commands, &cmder.Command{
45 | FlagSet: flagSet,
46 | Aliases: []string{},
47 | Handler: handler,
48 | UsageFunc: func() {
49 | fmt.Fprintf(flag.CommandLine.Output(), "Usage of 'doctree %s':\n", flagSet.Name())
50 | flagSet.PrintDefaults()
51 | fmt.Fprintf(flag.CommandLine.Output(), "%s", usage)
52 | },
53 | })
54 | }
55 |
--------------------------------------------------------------------------------
/cmd/doctree/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "flag"
5 | "os"
6 |
7 | "github.com/hexops/cmder"
8 |
9 | // Register language indexers.
10 | _ "github.com/sourcegraph/doctree/doctree/indexer/golang"
11 | _ "github.com/sourcegraph/doctree/doctree/indexer/javascript"
12 | _ "github.com/sourcegraph/doctree/doctree/indexer/markdown"
13 | _ "github.com/sourcegraph/doctree/doctree/indexer/python"
14 | _ "github.com/sourcegraph/doctree/doctree/indexer/zig"
15 | )
16 |
17 | // commands contains all registered subcommands.
18 | var commands cmder.Commander
19 |
20 | var usageText = `doctree is a tool for library documentation.
21 |
22 | Usage:
23 | doctree [arguments]
24 |
25 | The commands are:
26 | serve runs a doctree server
27 | index index a directory
28 | add (EXPERIMENTAL) register a directory for auto-indexing
29 |
30 | Use "doctree -h" for more information about a command.
31 | `
32 |
33 | func main() {
34 | commands.Run(flag.CommandLine, "doctree", usageText, os.Args[1:])
35 | }
36 |
--------------------------------------------------------------------------------
/cmd/doctree/search.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "flag"
6 | "fmt"
7 | "path/filepath"
8 |
9 | "github.com/hexops/cmder"
10 | "github.com/pkg/errors"
11 | "github.com/sourcegraph/doctree/doctree/indexer"
12 | )
13 |
14 | func init() {
15 | const usage = `
16 | Examples:
17 |
18 | Search :
19 |
20 | $ doctree search 'myquery'
21 |
22 | `
23 |
24 | // Parse flags for our subcommand.
25 | flagSet := flag.NewFlagSet("search", flag.ExitOnError)
26 | dataDirFlag := flagSet.String("data-dir", defaultDataDir(), "where doctree stores its data")
27 | projectNameFlag := flagSet.String("project", "", "search in a specific project")
28 |
29 | // Handles calls to our subcommand.
30 | handler := func(args []string) error {
31 | _ = flagSet.Parse(args)
32 | if flagSet.NArg() != 1 {
33 | return &cmder.UsageError{}
34 | }
35 | query := flagSet.Arg(0)
36 |
37 | ctx := context.Background()
38 | indexDataDir := filepath.Join(*dataDirFlag, "index")
39 | _, err := indexer.Search(ctx, indexDataDir, query, *projectNameFlag)
40 | if err != nil {
41 | return errors.Wrap(err, "Search")
42 | }
43 |
44 | // TODO: CLI interface for search! Print the results here at least :)
45 | return nil
46 | }
47 |
48 | // Register the command.
49 | commands = append(commands, &cmder.Command{
50 | FlagSet: flagSet,
51 | Aliases: []string{},
52 | Handler: handler,
53 | UsageFunc: func() {
54 | fmt.Fprintf(flag.CommandLine.Output(), "Usage of 'doctree %s':\n", flagSet.Name())
55 | flagSet.PrintDefaults()
56 | fmt.Fprintf(flag.CommandLine.Output(), "%s", usage)
57 | },
58 | })
59 | }
60 |
--------------------------------------------------------------------------------
/cmd/doctree/util.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "path/filepath"
7 | "strings"
8 |
9 | "github.com/fsnotify/fsnotify"
10 | "github.com/pkg/errors"
11 | "github.com/sourcegraph/doctree/doctree/git"
12 | )
13 |
14 | func defaultDataDir() string {
15 | home, err := os.UserHomeDir()
16 | if err != nil {
17 | fmt.Fprintf(os.Stderr, "warning: could not get home directory: %s", err)
18 | home = "."
19 | }
20 | return filepath.Join(home, ".doctree")
21 | }
22 |
23 | func defaultProjectName(defaultDir string) string {
24 | uri, err := git.URIForFile(defaultDir)
25 | if err != nil {
26 | absDir, err := filepath.Abs(defaultDir)
27 | if err != nil {
28 | return ""
29 | }
30 | return absDir
31 | }
32 | return uri
33 | }
34 |
35 | func isParentDir(parent, child string) (bool, error) {
36 | relativePath, err := filepath.Rel(parent, child)
37 | if err != nil {
38 | return false, err
39 | }
40 | return !strings.Contains(relativePath, ".."), nil
41 | }
42 |
43 | // Recursively watch a directory
44 | func recursiveWatch(watcher *fsnotify.Watcher, dir string) error {
45 | err := filepath.Walk(dir, func(walkPath string, fi os.FileInfo, err error) error {
46 | if err != nil {
47 | return err
48 | }
49 | if fi.IsDir() && !strings.HasPrefix(fi.Name(), ".") { // file is directory and isn't hidden
50 | if err = watcher.Add(walkPath); err != nil {
51 | return errors.Wrap(err, "watcher.Add")
52 | }
53 | }
54 | return nil
55 | })
56 | return err
57 | }
58 |
--------------------------------------------------------------------------------
/docs/development.md:
--------------------------------------------------------------------------------
1 | # Development
2 |
3 | ## Prerequisites
4 |
5 | * Go v1.18+
6 | * Zig v0.10+ ([nightly binary releases](https://ziglang.org/download/))
7 | * A recent / latest LTS version of [Node.js](https://nodejs.org/)
8 |
9 | [Task](https://taskfile.dev/#/installation) (alternative to `make` with file change watching):
10 |
11 | ```sh
12 | brew install go-task/tap/go-task
13 | ```
14 |
15 | Then ensure [Elm](https://elm-lang.org/), frontend dependencies and Zig libraries are cloned using:
16 |
17 | ```sh
18 | task setup
19 | ```
20 |
21 | ## Suggested tooling
22 |
23 | * For developing Elm frontend code, install [the "wrench" icon Elm plugin in VS Code](https://marketplace.visualstudio.com/items?itemName=Elmtooling.elm-ls-vscode)
24 | * For developing Zig code (unlikely), ensure you're using latest Zig version and [build zls from source](https://github.com/zigtools/zls#from-source), then install "ZLS for VSCode".
25 |
26 | ## Working with the code
27 |
28 | Just run `task` in the repository root and navigate to http://localhost:3333 - it'll do everything for you:
29 |
30 | * Build development tools for you (Go linter, gofumpt code formatter, etc.)
31 | * `go generate` any necessary code for you
32 | * Run Go linters and gofumpt code formatter for you.
33 | * Build and run `.bin/doctree serve` for you
34 |
35 | Best of all, it'll live reload the frontend code as you save changes in your editor. No need to even refresh the page!
36 |
37 | ## Sample repositories
38 |
39 | You can use the following to clone some sample repositories (into `../doctree-samples`) - useful for testing every language supported by Doctree:
40 |
41 | ```sh
42 | task dev-clone-sample-repos
43 | ```
44 |
45 | And then use the following to index them all:
46 |
47 | ```sh
48 | task dev-index-sample-repos
49 | ```
50 |
51 | ## Running tests
52 |
53 | You can use `task test` or `task test-race` (slower, but checks for race conditions).
54 |
55 | ## Building Docker image
56 |
57 | `task build-image` will build and tag a `sourcegraph/doctree:dev` image for you. `task run-image` will run it!
58 |
59 | ## Cross-compiling binaries for each OS
60 |
61 | If you have a macOS host machine you should be able to cross-compile binaries for each OS:
62 |
63 | ```
64 | task cross-compile
65 | ```
66 |
67 | Which should produce an `out/` directory with binaries for each OS.
68 |
69 | If not on macOS, you can use the `task cc-x86_64-linux` and `task cc-x86_64-windows` targets only for now.
70 |
--------------------------------------------------------------------------------
/docs/operations.md:
--------------------------------------------------------------------------------
1 | # Managing doctree.org
2 |
3 | For now, doctree.org is hosted in the Sourcegraph DigitalOcean account in [the doctree project](https://cloud.digitalocean.com/projects/00778e28-7044-4d03-9ab6-72b252afe76e/resources?i=2a039a) while things are so early stages / experimental. If you need access, let us know in the #doctree Slack channel.
4 |
5 | ## Adding a repository
6 |
7 | ```sh
8 | task ops-add-repo -- https://github.com/django/django
9 | ```
10 |
11 | ## Wipe all server data
12 |
13 | ```sh
14 | task ops-remove-all
15 | ```
16 |
17 | ## Restore our sample repositories
18 |
19 | ```sh
20 | task ops-add-sample-repos
21 | ```
22 |
23 | ## Deploy latest version
24 |
25 | ```sh
26 | task ops-deploy
27 | ```
28 |
--------------------------------------------------------------------------------
/doctree/apischema/apischema.go:
--------------------------------------------------------------------------------
1 | // Package apischema defines the JSON types that are returned by the doctree JSON API.
2 | package apischema
3 |
4 | import "github.com/sourcegraph/doctree/doctree/schema"
5 |
6 | // Page is the type returned by /api/get-page?project=github.com/sourcegraph/sourcegraph&language=markdown&page=/README.md
7 | type Page schema.Page
8 |
9 | // ProjectList is the type returned by /api/list
10 | type ProjectList []string
11 |
12 | // ProjectIndexes is the type returned by /api/get-index?name=github.com/sourcegraph/sourcegraph
13 | type ProjectIndexes map[string]schema.Index
14 |
15 | // SearchResults is the type returned by /api/search?query=foobar
16 | type SearchResults []SearchResult
17 |
18 | type SearchResult struct {
19 | Language string `json:"language"`
20 | ProjectName string `json:"projectName"`
21 | SearchKey string `json:"searchKey"`
22 | Path string `json:"path"`
23 | ID string `json:"id"`
24 | Score float64 `json:"score"`
25 | }
26 |
--------------------------------------------------------------------------------
/doctree/git/git.go:
--------------------------------------------------------------------------------
1 | package git
2 |
3 | import (
4 | "fmt"
5 | "net/url"
6 | "os/exec"
7 | "path/filepath"
8 | "strings"
9 |
10 | "github.com/pkg/errors"
11 | )
12 |
13 | func URIForFile(dir string) (string, error) {
14 | cmd := exec.Command("git", "config", "--get", "remote.origin.url")
15 | absDir, err := filepath.Abs(dir)
16 | if err != nil {
17 | return "", errors.Wrapf(err, "failed to get absolute path for %s", dir)
18 | }
19 | cmd.Dir = absDir
20 | out, err := cmd.Output()
21 | if err != nil {
22 | return "", errors.Wrapf(err, "git config --get remote.origin.url (pwd=%s)", cmd.Dir)
23 | }
24 | gitURL, err := normalizeGitURL(strings.TrimSpace(string(out)))
25 | if err != nil {
26 | return "", errors.Wrap(err, "normalizeGitURL")
27 | }
28 | return gitURL, nil
29 | }
30 |
31 | func normalizeGitURL(gitURL string) (string, error) {
32 | s := gitURL
33 | if strings.HasPrefix(gitURL, "git@") {
34 | s = strings.Replace(s, ":", "/", 1)
35 | s = strings.Replace(s, "git@", "git://", 1) // dummy scheme
36 | }
37 |
38 | u, err := url.Parse(s)
39 | if err != nil {
40 | return "", err
41 | }
42 | p := u.Path
43 | p = strings.TrimSuffix(p, ".git")
44 | p = strings.TrimPrefix(p, "/")
45 | return fmt.Sprintf("%s/%s", u.Hostname(), p), nil
46 | }
47 |
48 | func RevParse(dir string, abbrefRef bool, rev string) (string, error) {
49 | var cmd *exec.Cmd
50 | if abbrefRef {
51 | cmd = exec.Command("git", "rev-parse", "--abbrev-ref", rev)
52 | } else {
53 | cmd = exec.Command("git", "rev-parse", rev)
54 | }
55 | absDir, err := filepath.Abs(dir)
56 | if err != nil {
57 | return "", errors.Wrapf(err, "failed to get absolute path for %s", dir)
58 | }
59 | cmd.Dir = absDir
60 | out, err := cmd.Output()
61 | if err != nil {
62 | return "", errors.Wrapf(err, "git rev-parse ... (pwd=%s)", cmd.Dir)
63 | }
64 | return string(out), nil
65 | }
66 |
--------------------------------------------------------------------------------
/doctree/indexer/cli.go:
--------------------------------------------------------------------------------
1 | package indexer
2 |
3 | import (
4 | "encoding/json"
5 | "os"
6 | "path/filepath"
7 |
8 | "github.com/pkg/errors"
9 | )
10 |
11 | // Write autoindexedProjects as JSON in the provided filepath.
12 | func WriteAutoIndex(path string, autoindexedProjects map[string]AutoIndexedProject) error {
13 | f, err := os.Create(path)
14 | if err != nil {
15 | return errors.Wrap(err, "Create")
16 | }
17 | defer f.Close()
18 |
19 | if err := json.NewEncoder(f).Encode(autoindexedProjects); err != nil {
20 | return errors.Wrap(err, "Encode")
21 | }
22 |
23 | return nil
24 | }
25 |
26 | // Read autoindexedProjects array from the provided filepath.
27 | func ReadAutoIndex(path string) (map[string]AutoIndexedProject, error) {
28 | autoIndexedProjects := make(map[string]AutoIndexedProject)
29 | data, err := os.ReadFile(path)
30 | if err != nil {
31 | if _, err := os.Stat(filepath.Dir(path)); os.IsNotExist(err) {
32 | if err := os.Mkdir(filepath.Dir(path), os.ModePerm); err != nil {
33 | return nil, errors.Wrap(err, "CreateAutoIndexDirectory")
34 | }
35 | }
36 | if os.IsNotExist(err) {
37 | _, err := os.Create(path)
38 | if err != nil {
39 | return nil, errors.Wrap(err, "CreateAutoIndexFile")
40 | }
41 | return autoIndexedProjects, nil
42 | }
43 | return nil, errors.Wrap(err, "ReadAutoIndexFile")
44 | }
45 | err = json.Unmarshal(data, &autoIndexedProjects)
46 | if err != nil {
47 | return nil, errors.Wrap(err, "ParseAutoIndexFile")
48 | }
49 |
50 | return autoIndexedProjects, nil
51 | }
52 |
--------------------------------------------------------------------------------
/doctree/indexer/markdown/indexer.go:
--------------------------------------------------------------------------------
1 | // Package markdown provides a doctree indexer implementation for Markdown.
2 | package markdown
3 |
4 | import (
5 | "bytes"
6 | "context"
7 | "io/fs"
8 | "os"
9 | "path/filepath"
10 | "strings"
11 |
12 | "github.com/adrg/frontmatter"
13 | "github.com/pkg/errors"
14 | "github.com/sourcegraph/doctree/doctree/indexer"
15 | "github.com/sourcegraph/doctree/doctree/schema"
16 | )
17 |
18 | func init() {
19 | indexer.Register(&markdownIndexer{})
20 | }
21 |
22 | // Implements the indexer.Language interface.
23 | type markdownIndexer struct{}
24 |
25 | func (i *markdownIndexer) Name() schema.Language { return schema.LanguageMarkdown }
26 |
27 | func (i *markdownIndexer) Extensions() []string { return []string{"md"} }
28 |
29 | func (i *markdownIndexer) IndexDir(ctx context.Context, dir string) (*schema.Index, error) {
30 | // Find Go sources
31 | var sources []string
32 | if err := fs.WalkDir(os.DirFS(dir), ".", func(path string, d fs.DirEntry, err error) error {
33 | if err != nil {
34 | return err // error walking dir
35 | }
36 | if !d.IsDir() {
37 | ext := filepath.Ext(path)
38 | if ext == ".md" {
39 | sources = append(sources, path)
40 | }
41 | }
42 | return nil
43 | }); err != nil {
44 | return nil, errors.Wrap(err, "WalkDir")
45 | }
46 |
47 | files := 0
48 | bytes := 0
49 | pages := []schema.Page{}
50 | for _, path := range sources {
51 | dirFS := os.DirFS(dir)
52 | content, err := fs.ReadFile(dirFS, path)
53 | if err != nil {
54 | return nil, errors.Wrap(err, "ReadFile")
55 | }
56 | files += 1
57 | bytes += len(content)
58 |
59 | pages = append(pages, markdownToPage(content, path))
60 | }
61 |
62 | return &schema.Index{
63 | SchemaVersion: schema.LatestVersion,
64 | Language: schema.LanguageMarkdown,
65 | NumFiles: files,
66 | NumBytes: bytes,
67 | Libraries: []schema.Library{
68 | {
69 | Name: "TODO",
70 | ID: "TODO",
71 | Version: "TODO",
72 | VersionType: "TODO",
73 | Pages: pages,
74 | },
75 | },
76 | }, nil
77 | }
78 |
79 | func markdownToPage(content []byte, path string) schema.Page {
80 | // Strip frontmatter out of Markdown documents for now.
81 | var matter struct {
82 | Title string `yaml:"title"`
83 | Name string `yaml:"name"`
84 | Tags []string `yaml:"tags"`
85 | }
86 | rest, _ := frontmatter.Parse(bytes.NewReader(content), &matter)
87 |
88 | matterTitle := matter.Name
89 | if matterTitle == "" {
90 | matterTitle = matter.Title
91 | }
92 |
93 | primaryContent, childrenSections, firstHeaderName := markdownToSections(rest, 1, matterTitle)
94 |
95 | pageTitle := matterTitle
96 | if pageTitle == "" {
97 | pageTitle = firstHeaderName
98 | }
99 | searchKey := headerSearchKey(pageTitle, "")
100 | if pageTitle == "" {
101 | pageTitle = path
102 | // no search key for path, leave as empty list.
103 | }
104 | // TODO: Markdown headings often have Markdown/HTML images/links in them, we should strip those
105 | // out for the page title.
106 | if len(pageTitle) > 50 {
107 | pageTitle = pageTitle[:50]
108 | }
109 | return schema.Page{
110 | Path: path,
111 | Title: pageTitle,
112 | Detail: schema.Markdown(primaryContent),
113 | SearchKey: searchKey,
114 | Sections: childrenSections,
115 | }
116 | }
117 |
118 | func markdownToSections(content []byte, level int, pageTitle string) ([]byte, []schema.Section, string) {
119 | sectionPrefix := []byte(strings.Repeat("#", level) + " ")
120 |
121 | // Group all of the lines separated by a section prefix (e.g. "# heading 1")
122 | var sectionContent [][][]byte
123 | var lines [][]byte
124 | for _, line := range bytes.Split(content, []byte("\n")) {
125 | if bytes.HasPrefix(line, sectionPrefix) {
126 | if len(lines) > 0 {
127 | sectionContent = append(sectionContent, lines)
128 | }
129 | lines = nil
130 | }
131 | lines = append(lines, line)
132 | }
133 | if len(lines) > 0 {
134 | sectionContent = append(sectionContent, lines)
135 | }
136 |
137 | // Emit a section for each set of lines we accumulated.
138 | var (
139 | primaryContent []byte
140 | sections = []schema.Section{}
141 | firstHeaderName string
142 | )
143 | for _, lines := range sectionContent {
144 | var name string
145 | if bytes.HasPrefix(lines[0], sectionPrefix) {
146 | name = string(bytes.TrimPrefix(lines[0], sectionPrefix))
147 | }
148 |
149 | if level == 1 && name == "" {
150 | // This is the content before any heading in a document.
151 | subPrimaryContent, subChildrenSections, _ := markdownToSections(
152 | bytes.Join(lines, []byte("\n")),
153 | level+1,
154 | pageTitle,
155 | )
156 | primaryContent = subPrimaryContent
157 | sections = append(sections, subChildrenSections...)
158 | continue
159 | } else if name == "" {
160 | primaryContent = bytes.Join(lines, []byte("\n"))
161 | continue
162 | }
163 |
164 | if (level == 1) && firstHeaderName == "" {
165 | // This is the first header in a document. Elevate it out.
166 | firstHeaderName = name
167 | if pageTitle == "" {
168 | pageTitle = firstHeaderName
169 | }
170 | subPrimaryContent, subChildrenSections, _ := markdownToSections(
171 | bytes.Join(lines[1:], []byte("\n")),
172 | level+1,
173 | pageTitle,
174 | )
175 | primaryContent = subPrimaryContent
176 | sections = append(sections, subChildrenSections...)
177 | continue
178 | }
179 |
180 | subPrimaryContent, subChildrenSections, _ := markdownToSections(
181 | bytes.Join(lines[1:], []byte("\n")),
182 | level+1,
183 | pageTitle,
184 | )
185 |
186 | searchKey := headerSearchKey(pageTitle, name)
187 | if pageTitle == "" {
188 | searchKey = headerSearchKey(name, "")
189 | }
190 |
191 | sections = append(sections, schema.Section{
192 | ID: name,
193 | ShortLabel: name,
194 | Label: schema.Markdown(name),
195 | Detail: schema.Markdown(subPrimaryContent),
196 | SearchKey: searchKey,
197 | Children: subChildrenSections,
198 | })
199 | }
200 |
201 | if len(sections) == 0 && level < 6 {
202 | nonlinear := false
203 | for _, line := range bytes.Split(primaryContent, []byte("\n")) {
204 | if bytes.HasPrefix(line, []byte("#")) {
205 | nonlinear = true
206 | break
207 | }
208 | }
209 | if nonlinear {
210 | return markdownToSections(content, level+1, pageTitle)
211 | }
212 | }
213 | return primaryContent, sections, firstHeaderName
214 | }
215 |
216 | func headerSearchKey(pageTitle, section string) []string {
217 | name := joinNames(pageTitle, section)
218 | fields := strings.Fields(name)
219 | searchKey := make([]string, 0, 2+(len(fields)*2))
220 | searchKey = append(searchKey, []string{"#", " "}...)
221 | for i, field := range fields {
222 | searchKey = append(searchKey, field)
223 | if i != len(fields)-1 {
224 | searchKey = append(searchKey, " ")
225 | }
226 | }
227 | return searchKey
228 | }
229 |
230 | func joinNames(pageTitle, section string) string {
231 | limit := 60 - len("# ") - len(" > ")
232 | if len(pageTitle)+len(section) < limit {
233 | if section != "" {
234 | return pageTitle + " > " + section
235 | }
236 | return pageTitle
237 | }
238 | limit /= 2
239 | if len(pageTitle) > limit {
240 | pageTitle = pageTitle[:limit]
241 | }
242 | if len(section) > limit {
243 | section = section[:limit]
244 | }
245 | if section != "" {
246 | return pageTitle + " > " + section
247 | }
248 | return pageTitle
249 | }
250 |
--------------------------------------------------------------------------------
/doctree/indexer/markdown/indexer_test.go:
--------------------------------------------------------------------------------
1 | package markdown
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/hexops/autogold"
7 | "github.com/sourcegraph/doctree/doctree/schema"
8 | )
9 |
10 | func Test_markdownToPage_simple(t *testing.T) {
11 | page := markdownToPage([]byte(`# ziglearn
12 |
13 | Repo for https://ziglearn.org content. Feedback and PRs welcome.
14 |
15 | ## How to run the tests
16 |
17 | 1. `+"`"+`zig run test-out.zig`+"`"+`
18 | 2. `+"`"+`zig test do_tests.zig`+"`"+`
19 | `), "README.md")
20 |
21 | autogold.Want("simple", schema.Page{
22 | Path: "README.md", Title: "ziglearn", Detail: schema.Markdown(`
23 | Repo for https://ziglearn.org content. Feedback and PRs welcome.
24 | `),
25 | SearchKey: []string{
26 | "#",
27 | " ",
28 | "ziglearn",
29 | },
30 | Sections: []schema.Section{{
31 | ID: "How to run the tests",
32 | ShortLabel: "How to run the tests",
33 | Label: schema.Markdown("How to run the tests"),
34 | Detail: schema.Markdown("\n1. `zig run test-out.zig`\n2. `zig test do_tests.zig`\n"),
35 | SearchKey: []string{
36 | "#",
37 | " ",
38 | "ziglearn",
39 | " ",
40 | ">",
41 | " ",
42 | "How",
43 | " ",
44 | "to",
45 | " ",
46 | "run",
47 | " ",
48 | "the",
49 | " ",
50 | "tests",
51 | },
52 | Children: []schema.Section{},
53 | }},
54 | }).Equal(t, page)
55 | }
56 |
57 | func Test_markdownToPage_complex(t *testing.T) {
58 | page := markdownToPage([]byte(`# heading1
59 |
60 | content1
61 |
62 | ## heading2-0
63 |
64 | content2-0
65 |
66 | ### heading3-0
67 |
68 | content3-0
69 |
70 | #### heading4-0
71 |
72 | content4-0
73 |
74 | #### heading4-1
75 |
76 | content4-1
77 |
78 | ### heading3-1
79 |
80 | content3-1
81 |
82 | ## heading2-1
83 |
84 | content2-1
85 |
86 | `), "README.md")
87 |
88 | autogold.Want("simple", schema.Page{
89 | Path: "README.md", Title: "heading1", Detail: schema.Markdown("\ncontent1\n"),
90 | SearchKey: []string{
91 | "#",
92 | " ",
93 | "heading1",
94 | },
95 | Sections: []schema.Section{
96 | {
97 | ID: "heading2-0",
98 | ShortLabel: "heading2-0",
99 | Label: schema.Markdown("heading2-0"),
100 | Detail: schema.Markdown("\ncontent2-0\n"),
101 | SearchKey: []string{
102 | "#",
103 | " ",
104 | "heading1",
105 | " ",
106 | ">",
107 | " ",
108 | "heading2-0",
109 | },
110 | Children: []schema.Section{
111 | {
112 | ID: "heading3-0",
113 | ShortLabel: "heading3-0",
114 | Label: schema.Markdown("heading3-0"),
115 | Detail: schema.Markdown("\ncontent3-0\n"),
116 | SearchKey: []string{
117 | "#",
118 | " ",
119 | "heading1",
120 | " ",
121 | ">",
122 | " ",
123 | "heading3-0",
124 | },
125 | Children: []schema.Section{
126 | {
127 | ID: "heading4-0",
128 | ShortLabel: "heading4-0",
129 | Label: schema.Markdown("heading4-0"),
130 | Detail: schema.Markdown("\ncontent4-0\n"),
131 | SearchKey: []string{
132 | "#",
133 | " ",
134 | "heading1",
135 | " ",
136 | ">",
137 | " ",
138 | "heading4-0",
139 | },
140 | Children: []schema.Section{},
141 | },
142 | {
143 | ID: "heading4-1",
144 | ShortLabel: "heading4-1",
145 | Label: schema.Markdown("heading4-1"),
146 | Detail: schema.Markdown("\ncontent4-1\n"),
147 | SearchKey: []string{
148 | "#",
149 | " ",
150 | "heading1",
151 | " ",
152 | ">",
153 | " ",
154 | "heading4-1",
155 | },
156 | Children: []schema.Section{},
157 | },
158 | },
159 | },
160 | {
161 | ID: "heading3-1",
162 | ShortLabel: "heading3-1",
163 | Label: schema.Markdown("heading3-1"),
164 | Detail: schema.Markdown("\ncontent3-1\n"),
165 | SearchKey: []string{
166 | "#",
167 | " ",
168 | "heading1",
169 | " ",
170 | ">",
171 | " ",
172 | "heading3-1",
173 | },
174 | Children: []schema.Section{},
175 | },
176 | },
177 | },
178 | {
179 | ID: "heading2-1",
180 | ShortLabel: "heading2-1",
181 | Label: schema.Markdown("heading2-1"),
182 | Detail: schema.Markdown("\ncontent2-1\n\n"),
183 | SearchKey: []string{
184 | "#",
185 | " ",
186 | "heading1",
187 | " ",
188 | ">",
189 | " ",
190 | "heading2-1",
191 | },
192 | Children: []schema.Section{},
193 | },
194 | },
195 | }).Equal(t, page)
196 | }
197 |
198 | func Test_markdownToPage_frontmatter(t *testing.T) {
199 | page := markdownToPage([]byte(`
200 | ---
201 | title: "Mypage title"
202 | weight: 1
203 | date: 2021-01-23 20:52:00
204 | description: "yay"
205 | ---
206 |
207 | This content is not preceded by a heading.
208 |
209 | ## heading2
210 |
211 | content2
212 |
213 | `), "README.md")
214 |
215 | autogold.Want("simple", schema.Page{
216 | Path: "README.md", Title: "Mypage title",
217 | Detail: schema.Markdown(`
218 | This content is not preceded by a heading.
219 | `),
220 | SearchKey: []string{
221 | "#",
222 | " ",
223 | "Mypage",
224 | " ",
225 | "title",
226 | },
227 | Sections: []schema.Section{{
228 | ID: "heading2",
229 | ShortLabel: "heading2",
230 | Label: schema.Markdown("heading2"),
231 | Detail: schema.Markdown("\ncontent2\n\n"),
232 | SearchKey: []string{
233 | "#",
234 | " ",
235 | "Mypage",
236 | " ",
237 | "title",
238 | " ",
239 | ">",
240 | " ",
241 | "heading2",
242 | },
243 | Children: []schema.Section{},
244 | }},
245 | }).Equal(t, page)
246 | }
247 |
248 | func Test_markdownToPage_nonlinear_headers(t *testing.T) {
249 | page := markdownToPage([]byte(`# The Go Programming Language
250 | ### Download and Install
251 | #### Binary Distributions
252 | a
253 | #### Install From Source
254 | ### Contributing
255 | `), "README.md")
256 |
257 | autogold.Want("simple", schema.Page{
258 | Path: "README.md", Title: "The Go Programming Language",
259 | SearchKey: []string{
260 | "#",
261 | " ",
262 | "The",
263 | " ",
264 | "Go",
265 | " ",
266 | "Programming",
267 | " ",
268 | "Language",
269 | },
270 | Sections: []schema.Section{
271 | {
272 | ID: "Download and Install",
273 | ShortLabel: "Download and Install",
274 | Label: schema.Markdown("Download and Install"),
275 | SearchKey: []string{
276 | "#",
277 | " ",
278 | "The",
279 | " ",
280 | "Go",
281 | " ",
282 | "Programming",
283 | " ",
284 | "Language",
285 | " ",
286 | ">",
287 | " ",
288 | "Download",
289 | " ",
290 | "and",
291 | " ",
292 | "Install",
293 | },
294 | Children: []schema.Section{
295 | {
296 | ID: "Binary Distributions",
297 | ShortLabel: "Binary Distributions",
298 | Label: schema.Markdown("Binary Distributions"),
299 | Detail: schema.Markdown("a"),
300 | SearchKey: []string{
301 | "#",
302 | " ",
303 | "The",
304 | " ",
305 | "Go",
306 | " ",
307 | "Programming",
308 | " ",
309 | "Language",
310 | " ",
311 | ">",
312 | " ",
313 | "Binary",
314 | " ",
315 | "Distributions",
316 | },
317 | Children: []schema.Section{},
318 | },
319 | {
320 | ID: "Install From Source",
321 | ShortLabel: "Install From Source",
322 | Label: schema.Markdown("Install From Source"),
323 | SearchKey: []string{
324 | "#",
325 | " ",
326 | "The",
327 | " ",
328 | "Go",
329 | " ",
330 | "Programming",
331 | " ",
332 | "Language",
333 | " ",
334 | ">",
335 | " ",
336 | "Install",
337 | " ",
338 | "From",
339 | " ",
340 | "Source",
341 | },
342 | Children: []schema.Section{},
343 | },
344 | },
345 | },
346 | {
347 | ID: "Contributing",
348 | ShortLabel: "Contributing",
349 | Label: schema.Markdown("Contributing"),
350 | SearchKey: []string{
351 | "#",
352 | " ",
353 | "The",
354 | " ",
355 | "Go",
356 | " ",
357 | "Programming",
358 | " ",
359 | "Language",
360 | " ",
361 | ">",
362 | " ",
363 | "Contributing",
364 | },
365 | Children: []schema.Section{},
366 | },
367 | },
368 | }).Equal(t, page)
369 | }
370 |
371 | func Test_markdownToPage_starting_on_level_2(t *testing.T) {
372 | // Modeled after https://raw.githubusercontent.com/golang/go/master/src/cmd/compile/README.md
373 | page := markdownToPage([]byte(`
378 |
379 | ## Introduction to the Go compiler
380 |
381 | cmd/compile contains the main packages
382 |
383 | ### 1. Parsing
384 |
385 | yay
386 |
387 | `), "README.md")
388 |
389 | autogold.Want("simple", schema.Page{
390 | Path: "README.md", Title: "README.md", Detail: schema.Markdown(`
395 | `),
396 | // TODO: Should be first header in the file
397 | SearchKey: []string{
398 | "#",
399 | " ",
400 | },
401 | Sections: []schema.Section{{
402 | ID: "Introduction to the Go compiler",
403 | ShortLabel: "Introduction to the Go compiler",
404 | Label: schema.Markdown("Introduction to the Go compiler"),
405 | Detail: schema.Markdown("\ncmd/compile contains the main packages\n"),
406 | SearchKey: []string{
407 | "#",
408 | " ",
409 | "Introduction",
410 | " ",
411 | "to",
412 | " ",
413 | "the",
414 | " ",
415 | "Go",
416 | " ",
417 | "compiler",
418 | },
419 | Children: []schema.Section{{
420 | ID: "1. Parsing",
421 | ShortLabel: "1. Parsing",
422 | Label: schema.Markdown("1. Parsing"),
423 | Detail: schema.Markdown("\nyay\n\n"),
424 | SearchKey: []string{
425 | "#",
426 | " ",
427 | "1.",
428 | " ",
429 | "Parsing",
430 | },
431 | Children: []schema.Section{},
432 | }},
433 | }},
434 | }).Equal(t, page)
435 | }
436 |
--------------------------------------------------------------------------------
/doctree/indexer/python/indexer.go:
--------------------------------------------------------------------------------
1 | // Package python provides a doctree indexer implementation for Python.
2 | package python
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "io/fs"
8 | "os"
9 | "path/filepath"
10 | "strings"
11 |
12 | "github.com/pkg/errors"
13 | sitter "github.com/smacker/go-tree-sitter"
14 | "github.com/smacker/go-tree-sitter/python"
15 | "github.com/sourcegraph/doctree/doctree/indexer"
16 | "github.com/sourcegraph/doctree/doctree/schema"
17 | )
18 |
19 | func init() {
20 | indexer.Register(&pythonIndexer{})
21 | }
22 |
23 | // Implements the indexer.Language interface.
24 | type pythonIndexer struct{}
25 |
26 | func (i *pythonIndexer) Name() schema.Language { return schema.LanguagePython }
27 |
28 | func (i *pythonIndexer) Extensions() []string { return []string{"py", "py3"} }
29 |
30 | func (i *pythonIndexer) IndexDir(ctx context.Context, dir string) (*schema.Index, error) {
31 | // Find Python sources
32 | var sources []string
33 | if err := fs.WalkDir(os.DirFS(dir), ".", func(path string, d fs.DirEntry, err error) error {
34 | if err != nil {
35 | return err // error walking dir
36 | }
37 | if !d.IsDir() {
38 | ext := filepath.Ext(path)
39 | if ext == ".py" || ext == ".py3" {
40 | sources = append(sources, path)
41 | }
42 | }
43 | return nil
44 | }); err != nil {
45 | return nil, errors.Wrap(err, "WalkDir")
46 | }
47 |
48 | files := 0
49 | bytes := 0
50 | mods := map[string]moduleInfo{}
51 | functionsByMod := map[string][]schema.Section{}
52 | classesByMod := map[string][]schema.Section{}
53 |
54 | for _, path := range sources {
55 | if strings.Contains(path, "test_") || strings.Contains(path, "_test") || strings.Contains(path, "tests") {
56 | continue
57 | }
58 | dirFS := os.DirFS(dir)
59 | content, err := fs.ReadFile(dirFS, path)
60 | if err != nil {
61 | return nil, errors.Wrap(err, "ReadFile")
62 | }
63 |
64 | files += 1
65 | bytes += len(content)
66 |
67 | // Parse the file with tree-sitter.
68 | parser := sitter.NewParser()
69 | defer parser.Close()
70 | parser.SetLanguage(python.GetLanguage())
71 |
72 | tree, err := parser.ParseCtx(ctx, nil, content)
73 | if err != nil {
74 | return nil, errors.Wrap(err, "ParseCtx")
75 | }
76 | defer tree.Close()
77 |
78 | // Inspect the root node.
79 | n := tree.RootNode()
80 |
81 | // Module clauses
82 | var modName string
83 | {
84 | query, err := sitter.NewQuery([]byte(`
85 | (
86 | module
87 | .
88 | (comment)*
89 | .
90 | (expression_statement .
91 | (string) @module_docs
92 | )?
93 | )
94 | `), python.GetLanguage())
95 | if err != nil {
96 | return nil, errors.Wrap(err, "NewQuery")
97 | }
98 | defer query.Close()
99 |
100 | cursor := sitter.NewQueryCursor()
101 | defer cursor.Close()
102 | cursor.Exec(query, n)
103 |
104 | for {
105 | match, ok := cursor.NextMatch()
106 | if !ok {
107 | break
108 | }
109 | captures := getCaptures(query, match)
110 |
111 | // Extract module docs and Strip """ from both sides.
112 | modDocs := joinCaptures(content, captures["module_docs"], "\n")
113 | modDocs = sanitizeDocs(modDocs)
114 | modName = strings.ReplaceAll(strings.TrimSuffix(path, ".py"), "/", ".")
115 |
116 | mods[modName] = moduleInfo{path: path, docs: modDocs}
117 | }
118 | }
119 |
120 | funcDefQuery := `
121 | (
122 | function_definition
123 | name: (identifier) @func_name
124 | parameters: (parameters) @func_params
125 | return_type: (type)? @func_result
126 | body: (block . (expression_statement (string) @func_docs)?)
127 | )
128 | `
129 |
130 | // Function definitions
131 | {
132 | moduleFuncDefQuery := fmt.Sprintf("(module %s)", funcDefQuery)
133 | modFunctions, err := getFunctions(n, content, moduleFuncDefQuery, []string{modName})
134 | if err != nil {
135 | return nil, err
136 | }
137 |
138 | functionsByMod[modName] = modFunctions
139 | }
140 |
141 | // Classes and their methods
142 | {
143 | // Find out all the classes
144 | query, err := sitter.NewQuery([]byte(`
145 | (class_definition
146 | name: (identifier) @class_name
147 | superclasses: (argument_list)? @superclasses
148 | body: (block
149 | (expression_statement (string) @class_docs)?
150 | ) @class_body
151 | )
152 | `), python.GetLanguage())
153 | if err != nil {
154 | return nil, errors.Wrap(err, "NewQuery")
155 | }
156 | defer query.Close()
157 |
158 | cursor := sitter.NewQueryCursor()
159 | defer cursor.Close()
160 | cursor.Exec(query, n)
161 |
162 | // Iterate over the classes
163 | for {
164 | match, ok := cursor.NextMatch()
165 | if !ok {
166 | break
167 | }
168 | captures := getCaptures(query, match)
169 |
170 | className := firstCaptureContentOr(content, captures["class_name"], "")
171 | superClasses := firstCaptureContentOr(content, captures["superclasses"], "")
172 | classDocs := joinCaptures(content, captures["class_docs"], "\n")
173 | classDocs = sanitizeDocs(classDocs)
174 |
175 | classLabel := schema.Markdown("class " + className + superClasses)
176 | classes := classesByMod[modName]
177 |
178 | // Extract class methods:
179 | var classMethods []schema.Section
180 | classBodyNodes := captures["class_body"]
181 | if len(classBodyNodes) > 0 {
182 | classMethods, err = getFunctions(
183 | classBodyNodes[0], content, funcDefQuery,
184 | []string{modName, ".", className},
185 | )
186 | if err != nil {
187 | return nil, err
188 | }
189 | }
190 |
191 | classes = append(classes, schema.Section{
192 | ID: className,
193 | ShortLabel: className,
194 | Label: classLabel,
195 | Detail: schema.Markdown(classDocs),
196 | SearchKey: []string{modName, ".", className},
197 | Children: classMethods,
198 | })
199 | classesByMod[modName] = classes
200 | }
201 | }
202 | }
203 |
204 | var pages []schema.Page
205 | for modName, moduleInfo := range mods {
206 | functionsSection := schema.Section{
207 | ID: "func",
208 | ShortLabel: "func",
209 | Label: "Functions",
210 | SearchKey: []string{},
211 | Category: true,
212 | Children: functionsByMod[modName],
213 | }
214 |
215 | classesSection := schema.Section{
216 | ID: "class",
217 | ShortLabel: "class",
218 | Label: "Classes",
219 | SearchKey: []string{},
220 | Category: true,
221 | Children: classesByMod[modName],
222 | }
223 |
224 | pages = append(pages, schema.Page{
225 | Path: moduleInfo.path,
226 | Title: "Module " + modName,
227 | Detail: schema.Markdown(moduleInfo.docs),
228 | SearchKey: []string{modName},
229 | Sections: []schema.Section{functionsSection, classesSection},
230 | })
231 | }
232 |
233 | return &schema.Index{
234 | SchemaVersion: schema.LatestVersion,
235 | Language: schema.LanguagePython,
236 | NumFiles: files,
237 | NumBytes: bytes,
238 | Libraries: []schema.Library{
239 | {
240 | Name: "TODO",
241 | ID: "TODO",
242 | Version: "TODO",
243 | VersionType: "TODO",
244 | Pages: pages,
245 | },
246 | },
247 | }, nil
248 | }
249 |
250 | func getFunctions(node *sitter.Node, content []byte, q string, searchKeyPrefix []string) ([]schema.Section, error) {
251 | var functions []schema.Section
252 | query, err := sitter.NewQuery([]byte(q), python.GetLanguage())
253 | if err != nil {
254 | return nil, errors.Wrap(err, "NewQuery")
255 | }
256 | defer query.Close()
257 |
258 | cursor := sitter.NewQueryCursor()
259 | defer cursor.Close()
260 | cursor.Exec(query, node)
261 |
262 | for {
263 | match, ok := cursor.NextMatch()
264 | if !ok {
265 | break
266 | }
267 | captures := getCaptures(query, match)
268 | funcDocs := joinCaptures(content, captures["func_docs"], "\n")
269 | funcDocs = sanitizeDocs(funcDocs)
270 | funcName := firstCaptureContentOr(content, captures["func_name"], "")
271 | funcParams := firstCaptureContentOr(content, captures["func_params"], "")
272 | funcResult := firstCaptureContentOr(content, captures["func_result"], "")
273 |
274 | if len(funcName) > 0 && funcName[0] == '_' && funcName[len(funcName)-1] != '_' {
275 | continue // unexported (private function)
276 | }
277 |
278 | funcLabel := schema.Markdown("def " + funcName + funcParams)
279 | if funcResult != "" {
280 | funcLabel = funcLabel + schema.Markdown(" -> "+funcResult)
281 | }
282 | functions = append(functions, schema.Section{
283 | ID: funcName,
284 | ShortLabel: funcName,
285 | Label: funcLabel,
286 | Detail: schema.Markdown(funcDocs),
287 | SearchKey: append(searchKeyPrefix, ".", funcName),
288 | })
289 | }
290 |
291 | return functions, nil
292 | }
293 |
294 | func sanitizeDocs(s string) string {
295 | return strings.TrimSuffix(strings.TrimPrefix(s, "\"\"\""), "\"\"\"")
296 | }
297 |
298 | type moduleInfo struct {
299 | path string
300 | docs string
301 | }
302 |
303 | func firstCaptureContentOr(content []byte, captures []*sitter.Node, defaultValue string) string {
304 | if len(captures) > 0 {
305 | return captures[0].Content(content)
306 | }
307 | return defaultValue
308 | }
309 |
310 | func joinCaptures(content []byte, captures []*sitter.Node, sep string) string {
311 | var v []string
312 | for _, capture := range captures {
313 | v = append(v, capture.Content(content))
314 | }
315 | return strings.Join(v, sep)
316 | }
317 |
318 | func getCaptures(q *sitter.Query, m *sitter.QueryMatch) map[string][]*sitter.Node {
319 | captures := map[string][]*sitter.Node{}
320 | for _, c := range m.Captures {
321 | cname := q.CaptureNameForId(c.Index)
322 | captures[cname] = append(captures[cname], c.Node)
323 | }
324 | return captures
325 | }
326 |
--------------------------------------------------------------------------------
/doctree/indexer/search.go:
--------------------------------------------------------------------------------
1 | package indexer
2 |
3 | import (
4 | "bytes"
5 | "context"
6 | "encoding/gob"
7 | "fmt"
8 | "io/ioutil"
9 | "log"
10 | "os"
11 | "path/filepath"
12 | "sort"
13 | "strings"
14 | "time"
15 |
16 | "github.com/agnivade/levenshtein"
17 | "github.com/pkg/errors"
18 | "github.com/sourcegraph/doctree/doctree/apischema"
19 | "github.com/sourcegraph/doctree/doctree/schema"
20 | sinter "github.com/sourcegraph/doctree/libs/sinter/bindings/sinter-go"
21 | "github.com/spaolacci/murmur3"
22 | )
23 |
24 | // IndexForSearch produces search indexes for the given project, writing them to:
25 | //
26 | // index//search-index.sinter
27 | func IndexForSearch(projectName, indexDataDir string, indexes map[string]*schema.Index) error {
28 | start := time.Now()
29 | filter, err := sinter.FilterInit(10_000_000)
30 | if err != nil {
31 | return errors.Wrap(err, "FilterInit")
32 | }
33 | defer filter.Deinit()
34 |
35 | walkPage := func(p schema.Page, keys [][]string, ids []string) ([][]string, []string) {
36 | keys = append(keys, p.SearchKey)
37 | ids = append(ids, "")
38 |
39 | var walkSection func(s schema.Section)
40 | walkSection = func(s schema.Section) {
41 | keys = append(keys, s.SearchKey)
42 | ids = append(ids, s.ID)
43 |
44 | for _, child := range s.Children {
45 | walkSection(child)
46 | }
47 | }
48 | for _, section := range p.Sections {
49 | walkSection(section)
50 | }
51 | return keys, ids
52 | }
53 |
54 | totalNumKeys := 0
55 | totalNumSearchKeys := 0
56 | insert := func(language, projectName, pagePath string, searchKeys [][]string, ids []string) error {
57 | absoluteKeys := make([][]string, 0, len(searchKeys))
58 | for _, searchKey := range searchKeys {
59 | absoluteKeys = append(absoluteKeys, append([]string{language, projectName}, searchKey...))
60 | }
61 | if len(absoluteKeys) != len(ids) {
62 | panic("invariant: len(absoluteKeys) != len(ids)")
63 | }
64 |
65 | totalNumSearchKeys += len(searchKeys)
66 | fuzzyKeys := fuzzyKeys(absoluteKeys)
67 | totalNumKeys += len(fuzzyKeys)
68 |
69 | var buf bytes.Buffer
70 | enc := gob.NewEncoder(&buf)
71 | if err := enc.Encode(sinterResult{
72 | Language: language,
73 | ProjectName: projectName,
74 | SearchKeys: searchKeys,
75 | IDs: ids,
76 | Path: pagePath,
77 | }); err != nil {
78 | return errors.Wrap(err, "Encode")
79 | }
80 |
81 | if err := filter.Insert(&sinter.SliceIterator{Slice: fuzzyKeys}, buf.Bytes()); err != nil {
82 | return errors.Wrap(err, "Insert")
83 | }
84 | return nil
85 | }
86 |
87 | for language, index := range indexes {
88 | for _, lib := range index.Libraries {
89 | for _, page := range lib.Pages {
90 | searchKeys, ids := walkPage(page, nil, nil)
91 | if err := insert(language, projectName, page.Path, searchKeys, ids); err != nil {
92 | return err
93 | }
94 | for _, subPage := range page.Subpages {
95 | searchKeys, ids := walkPage(subPage, nil, nil)
96 | if err := insert(language, projectName, page.Path, searchKeys, ids); err != nil {
97 | return err
98 | }
99 | }
100 | }
101 | }
102 | }
103 |
104 | if err := filter.Index(); err != nil {
105 | return errors.Wrap(err, "Index")
106 | }
107 |
108 | indexDataDir, err = filepath.Abs(indexDataDir)
109 | if err != nil {
110 | return errors.Wrap(err, "Abs")
111 | }
112 | outDir := filepath.Join(indexDataDir, encodeProjectName(projectName))
113 | if err := os.MkdirAll(outDir, os.ModePerm); err != nil {
114 | return errors.Wrap(err, "MkdirAll")
115 | }
116 |
117 | if err := filter.WriteFile(filepath.Join(outDir, "search-index.sinter")); err != nil {
118 | return errors.Wrap(err, "WriteFile")
119 | }
120 | // TODO: This should be in cmd/doctree, not here.
121 | fmt.Printf("search: indexed %v filter keys (%v search keys) in %v\n", totalNumKeys, totalNumSearchKeys, time.Since(start))
122 |
123 | return nil
124 | }
125 |
126 | func Search(ctx context.Context, indexDataDir, query, projectName string) (apischema.SearchResults, error) {
127 | query, language := parseQuery(query)
128 |
129 | // TODO: could skip sinter filter indexes from projects without our desired language.
130 | var indexes []string
131 | if projectName == "" {
132 | dir, err := ioutil.ReadDir(indexDataDir)
133 | if os.IsNotExist(err) {
134 | return apischema.SearchResults{}, nil
135 | }
136 | if err != nil {
137 | return nil, errors.Wrap(err, "ReadDir")
138 | }
139 | for _, info := range dir {
140 | if info.IsDir() {
141 | indexes = append(indexes, filepath.Join(indexDataDir, info.Name(), "search-index.sinter"))
142 | }
143 | }
144 | } else {
145 | indexes = append(indexes, filepath.Join(
146 | indexDataDir,
147 | encodeProjectName(projectName),
148 | "search-index.sinter",
149 | ))
150 | }
151 |
152 | queryKey := strings.FieldsFunc(query, func(r rune) bool { return r == '.' || r == '/' || r == ' ' })
153 | queryKeyHashes := []uint64{}
154 | for _, part := range queryKey {
155 | queryKeyHashes = append(queryKeyHashes, hash(part))
156 | }
157 | if len(queryKeyHashes) == 0 {
158 | // TODO: make QueryLogicalOr handle empty keys set
159 | queryKeyHashes = []uint64{hash(query)}
160 | }
161 |
162 | // TODO: return stats about search performance, etc.
163 | // TODO: query limiting support
164 | // TODO: support filtering to specific project
165 | const rankedResultLimit = 10000
166 | const limit = 100
167 | out := apischema.SearchResults{}
168 | for _, sinterFile := range indexes {
169 | sinterFilter, err := sinter.FilterReadFile(sinterFile)
170 | if err != nil {
171 | log.Println("error searching", sinterFile, "FilterReadFile:", err)
172 | continue
173 | }
174 |
175 | results, err := sinterFilter.QueryLogicalOr(queryKeyHashes)
176 | if err != nil {
177 | log.Println("error searching", sinterFile, "QueryLogicalOr:", err)
178 | continue
179 | }
180 | defer results.Deinit()
181 |
182 | out = append(out, decodeResults(results, queryKey, language, rankedResultLimit-len(out))...)
183 | if len(out) >= rankedResultLimit {
184 | break
185 | }
186 | }
187 | sort.Slice(out, func(i, j int) bool { return out[i].Score > out[j].Score })
188 | if len(out) > limit {
189 | out = out[:limit]
190 | }
191 | return out, nil
192 | }
193 |
194 | var languageSearchTerms = map[string]schema.Language{
195 | "cpp": schema.LanguageCpp,
196 | "c++": schema.LanguageCpp,
197 | "cxx": schema.LanguageCpp,
198 | "go": schema.LanguageGo,
199 | "golang": schema.LanguageGo,
200 | "java": schema.LanguageJava,
201 | "objc": schema.LanguageObjC,
202 | "python": schema.LanguagePython,
203 | "py": schema.LanguagePython,
204 | "typescript": schema.LanguageTypeScript,
205 | "ts": schema.LanguageTypeScript,
206 | "zig": schema.LanguageZig,
207 | "ziglang": schema.LanguageZig,
208 | "markdown": schema.LanguageMarkdown,
209 | "md": schema.LanguageMarkdown,
210 | }
211 |
212 | // Examples:
213 | //
214 | // "foo bar" -> ("foo bar", nil)
215 | // "gofoo bar" -> ("gofoo bar", nil)
216 | //
217 | // "go foo bar" -> ("foo bar", schema.LanguageGo)
218 | // "foo bar c++" -> ("foo bar", schema.LanguageCpp)
219 | //
220 | // "go foo bar java" -> ("foo bar java", schema.LanguageGo)
221 | // " go foo bar" -> ("go foo bar", nil)
222 | // "foo bar java " -> ("foo bar java", nil)
223 | //
224 | func parseQuery(query string) (realQuery string, language *schema.Language) {
225 | // If the query starts with a known language term, we use that first.
226 | for term, lang := range languageSearchTerms {
227 | if strings.HasPrefix(query, term+" ") {
228 | return strings.TrimPrefix(query, term+" "), &lang
229 | }
230 | }
231 |
232 | // Secondarily, if the query ends with a known language term we use that.
233 | for term, lang := range languageSearchTerms {
234 | if strings.HasSuffix(query, " "+term) {
235 | return strings.TrimSuffix(query, " "+term), &lang
236 | }
237 | }
238 | return query, nil
239 | }
240 |
241 | type sinterResult struct {
242 | Language string `json:"language"`
243 | ProjectName string `json:"projectName"`
244 | SearchKeys [][]string `json:"searchKeys"`
245 | IDs []string `json:"ids"`
246 | Path string `json:"path"`
247 | }
248 |
249 | func decodeResults(results sinter.FilterResults, queryKey []string, language *schema.Language, limit int) apischema.SearchResults {
250 | var out apischema.SearchResults
251 | decoding:
252 | for i := 0; i < results.Len(); i++ {
253 | var result sinterResult
254 | err := gob.NewDecoder(bytes.NewReader(results.Index(i))).Decode(&result)
255 | if err != nil {
256 | panic("illegal sinter result value: " + err.Error())
257 | }
258 |
259 | if language != nil && result.Language != language.ID {
260 | continue
261 | }
262 | for index, searchKey := range result.SearchKeys {
263 | absoluteKey := append([]string{result.Language, result.ProjectName}, searchKey...)
264 | score := match(queryKey, absoluteKey)
265 | if score > 0.5 {
266 | out = append(out, apischema.SearchResult{
267 | Language: result.Language,
268 | ProjectName: result.ProjectName,
269 | SearchKey: strings.Join(searchKey, ""),
270 | Path: result.Path,
271 | ID: result.IDs[index],
272 | Score: score,
273 | })
274 | if len(out) >= limit {
275 | break decoding
276 | }
277 | }
278 | }
279 | }
280 | return out
281 | }
282 |
283 | func match(queryKey []string, key []string) float64 {
284 | matchThreshold := 0.75
285 |
286 | score := 0.0
287 | lastScore := 0.0
288 | for _, queryPart := range queryKey {
289 | queryPart = strings.ToLower(queryPart)
290 | for i, keyPart := range key {
291 | keyPart = strings.ToLower(keyPart)
292 | largest := len(queryPart)
293 | if len(keyPart) > largest {
294 | largest = len(keyPart)
295 | }
296 | // [1.0, 0.0] where 1.0 is exactly equal
297 | partScore := 1.0 - (float64(levenshtein.ComputeDistance(queryPart, keyPart)) / float64(largest))
298 |
299 | boost := float64(len(key) - i) // Matches on left side of key get more boost
300 | if partScore > matchThreshold && lastScore > matchThreshold {
301 | boost *= 2
302 | }
303 | finalPartScore := partScore * boost
304 | score += finalPartScore
305 | lastScore = finalPartScore
306 | }
307 | }
308 | return score
309 | }
310 |
311 | func fuzzyKeys(keys [][]string) []uint64 {
312 | var fuzzyKeys []uint64
313 | for _, wholeKey := range keys {
314 | for _, part := range wholeKey {
315 | runes := []rune(part)
316 | fuzzyKeys = append(fuzzyKeys, prefixKeys(runes)...)
317 | fuzzyKeys = append(fuzzyKeys, suffixKeys(runes)...)
318 | lowerRunes := []rune(strings.ToLower(part))
319 | fuzzyKeys = append(fuzzyKeys, prefixKeys(lowerRunes)...)
320 | fuzzyKeys = append(fuzzyKeys, suffixKeys(lowerRunes)...)
321 | }
322 | }
323 | return fuzzyKeys
324 | }
325 |
326 | func prefixKeys(runes []rune) []uint64 {
327 | var keys []uint64
328 | var prefix []rune
329 | for _, r := range runes {
330 | prefix = append(prefix, r)
331 | keys = append(keys, hash(string(prefix)))
332 | }
333 | return keys
334 | }
335 |
336 | func suffixKeys(runes []rune) []uint64 {
337 | var keys []uint64
338 | var suffix []rune
339 | for i := len(runes) - 1; i >= 0; i-- {
340 | suffix = append([]rune{runes[i]}, suffix...)
341 | keys = append(keys, hash(string(suffix)))
342 | }
343 | return keys
344 | }
345 |
346 | func hash(s string) uint64 {
347 | return murmur3.Sum64([]byte(s))
348 | }
349 |
--------------------------------------------------------------------------------
/doctree/indexer/zig/indexer.go:
--------------------------------------------------------------------------------
1 | // Package zig provides a doctree indexer implementation for Zig.
2 | package zig
3 |
4 | import (
5 | "context"
6 | "io/fs"
7 | "os"
8 | "path"
9 | "path/filepath"
10 | "strings"
11 |
12 | "github.com/pkg/errors"
13 | zig "github.com/slimsag/tree-sitter-zig/bindings/go"
14 | sitter "github.com/smacker/go-tree-sitter"
15 | "github.com/sourcegraph/doctree/doctree/indexer"
16 | "github.com/sourcegraph/doctree/doctree/schema"
17 | )
18 |
19 | func init() {
20 | indexer.Register(&zigIndexer{})
21 | }
22 |
23 | // Implements the indexer.Language interface.
24 | type zigIndexer struct{}
25 |
26 | func (i *zigIndexer) Name() schema.Language { return schema.LanguageZig }
27 |
28 | func (i *zigIndexer) Extensions() []string { return []string{"zig"} }
29 |
30 | func (i *zigIndexer) IndexDir(ctx context.Context, dir string) (*schema.Index, error) {
31 | // Find Zig sources
32 | var sources []string
33 | dirFS := os.DirFS(dir)
34 | if err := fs.WalkDir(dirFS, ".", func(path string, d fs.DirEntry, err error) error {
35 | if err != nil {
36 | return err // error walking dir
37 | }
38 | if !d.IsDir() {
39 | ext := filepath.Ext(path)
40 | if ext == ".zig" {
41 | sources = append(sources, path)
42 | }
43 | }
44 | return nil
45 | }); err != nil {
46 | return nil, errors.Wrap(err, "WalkDir")
47 | }
48 |
49 | deps := depGraph{}
50 | for _, path := range sources {
51 | content, err := fs.ReadFile(dirFS, path)
52 | if err != nil {
53 | return nil, errors.Wrap(err, "ReadFile")
54 | }
55 |
56 | // Parse the file with tree-sitter.
57 | parser := sitter.NewParser()
58 | defer parser.Close()
59 | parser.SetLanguage(zig.GetLanguage())
60 |
61 | tree, err := parser.ParseCtx(ctx, nil, content)
62 | if err != nil {
63 | return nil, errors.Wrap(err, "ParseCtx")
64 | }
65 | defer tree.Close()
66 |
67 | // Inspect the root node.
68 | n := tree.RootNode()
69 |
70 | // Variable declarations
71 | {
72 | query, err := sitter.NewQuery([]byte(`
73 | (
74 | "pub"? @pub
75 | .
76 | (TopLevelDecl
77 | (VarDecl
78 | variable_type_function:
79 | (IDENTIFIER) @var_name
80 | (ErrorUnionExpr
81 | (SuffixExpr
82 | (BUILTINIDENTIFIER)
83 | (FnCallArguments
84 | (ErrorUnionExpr
85 | (SuffixExpr
86 | (STRINGLITERALSINGLE)
87 | )
88 | )
89 | )
90 | )
91 | ) @var_expr
92 | )
93 | )
94 | )
95 | `), zig.GetLanguage())
96 | if err != nil {
97 | return nil, errors.Wrap(err, "NewQuery")
98 | }
99 | defer query.Close()
100 |
101 | cursor := sitter.NewQueryCursor()
102 | defer cursor.Close()
103 | cursor.Exec(query, n)
104 |
105 | for {
106 | match, ok := cursor.NextMatch()
107 | if !ok {
108 | break
109 | }
110 | captures := getCaptures(query, match)
111 |
112 | pub := firstCaptureContentOr(content, captures["pub"], "") == "pub"
113 | varName := firstCaptureContentOr(content, captures["var_name"], "")
114 | varExpr := firstCaptureContentOr(content, captures["var_expr"], "")
115 |
116 | if strings.HasPrefix(varExpr, "@import(") {
117 | importPath := strings.TrimSuffix(strings.TrimPrefix(varExpr, `@import("`), `")`)
118 | deps.insert(path, pub, varName, importPath)
119 | }
120 | }
121 | }
122 | }
123 | deps.build()
124 |
125 | files := 0
126 | bytes := 0
127 | functionsByFile := map[string][]schema.Section{}
128 | for _, path := range sources {
129 | content, err := fs.ReadFile(dirFS, path)
130 | if err != nil {
131 | return nil, errors.Wrap(err, "ReadFile")
132 | }
133 | files += 1
134 | bytes += len(content)
135 |
136 | // Parse the file with tree-sitter.
137 | parser := sitter.NewParser()
138 | defer parser.Close()
139 | parser.SetLanguage(zig.GetLanguage())
140 |
141 | tree, err := parser.ParseCtx(ctx, nil, content)
142 | if err != nil {
143 | return nil, errors.Wrap(err, "ParseCtx")
144 | }
145 | defer tree.Close()
146 |
147 | // Inspect the root node.
148 | n := tree.RootNode()
149 |
150 | // Variable declarations
151 | {
152 | query, err := sitter.NewQuery([]byte(`
153 | (
154 | (container_doc_comment)* @container_docs
155 | .
156 | "pub"? @pub
157 | .
158 | (TopLevelDecl
159 | (VarDecl
160 | variable_type_function:
161 | (IDENTIFIER) @var_name
162 | (ErrorUnionExpr
163 | (SuffixExpr
164 | (BUILTINIDENTIFIER)
165 | (FnCallArguments
166 | (ErrorUnionExpr
167 | (SuffixExpr
168 | (STRINGLITERALSINGLE)
169 | )
170 | )
171 | )
172 | )
173 | ) @var_expr
174 | )
175 | )
176 | )
177 | `), zig.GetLanguage())
178 | if err != nil {
179 | return nil, errors.Wrap(err, "NewQuery")
180 | }
181 | defer query.Close()
182 |
183 | cursor := sitter.NewQueryCursor()
184 | defer cursor.Close()
185 | cursor.Exec(query, n)
186 |
187 | for {
188 | match, ok := cursor.NextMatch()
189 | if !ok {
190 | break
191 | }
192 | captures := getCaptures(query, match)
193 |
194 | containerDocs := firstCaptureContentOr(content, captures["container_docs"], "")
195 | pub := firstCaptureContentOr(content, captures["pub"], "") == "pub"
196 | varName := firstCaptureContentOr(content, captures["var_name"], "")
197 | varExpr := firstCaptureContentOr(content, captures["var_expr"], "")
198 |
199 | _ = containerDocs
200 | _ = pub
201 | _ = varName
202 | _ = varExpr
203 | // TODO: emit variables/constants section
204 | }
205 | }
206 |
207 | // Function definitions
208 | {
209 | // TODO: This query is incorrectly pulling out methods from nested struct definitions.
210 | // So we end up with a flat hierarchy of types - that's very bad. It also means we don't
211 | // accurately pick up when a method is part of a parent type.
212 | query, err := sitter.NewQuery([]byte(`
213 | (
214 | (doc_comment)* @func_docs
215 | .
216 | "pub"? @pub
217 | .
218 | (TopLevelDecl
219 | (FnProto
220 | function:
221 | (IDENTIFIER) @func_name
222 | (ParamDeclList) @func_params
223 | (ErrorUnionExpr
224 | (SuffixExpr
225 | (BuildinTypeExpr)
226 | )
227 | ) @func_result
228 | )
229 | )
230 | )
231 | `), zig.GetLanguage())
232 | if err != nil {
233 | return nil, errors.Wrap(err, "NewQuery")
234 | }
235 | defer query.Close()
236 |
237 | cursor := sitter.NewQueryCursor()
238 | defer cursor.Close()
239 | cursor.Exec(query, n)
240 |
241 | for {
242 | match, ok := cursor.NextMatch()
243 | if !ok {
244 | break
245 | }
246 | captures := getCaptures(query, match)
247 |
248 | pub := firstCaptureContentOr(content, captures["pub"], "")
249 | if pub != "pub" {
250 | continue
251 | }
252 |
253 | funcDocs := firstCaptureContentOr(content, captures["func_docs"], "")
254 | funcName := firstCaptureContentOr(content, captures["func_name"], "")
255 | funcParams := firstCaptureContentOr(content, captures["func_params"], "")
256 | funcResult := firstCaptureContentOr(content, captures["func_result"], "")
257 |
258 | accessiblePath := deps.fileToAccessiblePath[path]
259 | var searchKey []string
260 | if accessiblePath == "" {
261 | searchKey = []string{funcName}
262 | } else {
263 | searchKey = []string{accessiblePath, ".", funcName}
264 | }
265 | functionsByFile[path] = append(functionsByFile[path], schema.Section{
266 | ID: funcName,
267 | ShortLabel: funcName,
268 | Label: schema.Markdown(funcName + funcParams + " " + funcResult),
269 | Detail: schema.Markdown(docsToMarkdown(funcDocs)),
270 | SearchKey: searchKey,
271 | })
272 | }
273 | }
274 | }
275 |
276 | var pages []schema.Page
277 | for path, functions := range functionsByFile {
278 | functionsSection := schema.Section{
279 | ID: "fn",
280 | ShortLabel: "fn",
281 | Label: "Functions",
282 | Category: true,
283 | SearchKey: []string{},
284 | Children: functions,
285 | }
286 |
287 | pages = append(pages, schema.Page{
288 | Path: path,
289 | Title: path,
290 | Detail: schema.Markdown("TODO"),
291 | SearchKey: []string{path},
292 | Sections: []schema.Section{functionsSection},
293 | })
294 | }
295 |
296 | return &schema.Index{
297 | SchemaVersion: schema.LatestVersion,
298 | Language: schema.LanguageZig,
299 | NumFiles: files,
300 | NumBytes: bytes,
301 | Libraries: []schema.Library{
302 | {
303 | Name: "TODO",
304 | ID: "TODO",
305 | Version: "TODO",
306 | VersionType: "TODO",
307 | Pages: pages,
308 | },
309 | },
310 | }, nil
311 | }
312 |
313 | type importRecord struct {
314 | path string
315 | pub bool
316 | name string
317 | importPath string
318 | }
319 |
320 | type depGraph struct {
321 | records []importRecord
322 | fileToAccessiblePath map[string]string
323 | }
324 |
325 | func (d *depGraph) insert(path string, pub bool, name, importPath string) {
326 | d.records = append(d.records, importRecord{path, pub, name, importPath})
327 | if d.fileToAccessiblePath == nil {
328 | d.fileToAccessiblePath = map[string]string{}
329 | }
330 | d.fileToAccessiblePath[path] = ""
331 | }
332 |
333 | func (d *depGraph) build() {
334 | for filePath := range d.fileToAccessiblePath {
335 | path := d.collect(filePath, nil, map[string]struct{}{})
336 | d.fileToAccessiblePath[filePath] = strings.Join(path, ".")
337 | // fmt.Println(filePath, strings.Join(path, "."))
338 | }
339 | }
340 |
341 | func (d *depGraph) collect(targetPath string, result []string, cyclic map[string]struct{}) []string {
342 | for _, record := range d.records {
343 | if !record.pub {
344 | continue
345 | }
346 | if strings.HasSuffix(record.importPath, ".zig") {
347 | record.importPath = path.Join(path.Dir(record.path), record.importPath)
348 | }
349 | if record.importPath == targetPath {
350 | if _, ok := cyclic[record.path]; ok {
351 | return result
352 | }
353 | cyclic[record.path] = struct{}{}
354 | return d.collect(record.path, append([]string{record.name}, result...), cyclic)
355 | }
356 | }
357 | return result
358 | }
359 |
360 | func docsToMarkdown(docs string) string {
361 | var out []string
362 | for _, s := range strings.Split(docs, "\n") {
363 | if strings.HasPrefix(s, "/// ") {
364 | out = append(out, strings.TrimPrefix(s, "/// "))
365 | continue
366 | } else if strings.HasPrefix(s, "//! ") {
367 | out = append(out, strings.TrimPrefix(s, "//! "))
368 | continue
369 | }
370 | out = append(out, strings.TrimPrefix(s, "// "))
371 | }
372 | return strings.Join(out, "\n")
373 | }
374 |
375 | func firstCaptureContentOr(content []byte, captures []*sitter.Node, defaultValue string) string {
376 | if len(captures) > 0 {
377 | return captures[0].Content(content)
378 | }
379 | return defaultValue
380 | }
381 |
382 | func getCaptures(q *sitter.Query, m *sitter.QueryMatch) map[string][]*sitter.Node {
383 | captures := map[string][]*sitter.Node{}
384 | for _, c := range m.Captures {
385 | cname := q.CaptureNameForId(c.Index)
386 | captures[cname] = append(captures[cname], c.Node)
387 | }
388 | return captures
389 | }
390 |
--------------------------------------------------------------------------------
/doctree/schema/schema.go:
--------------------------------------------------------------------------------
1 | // Package schema describes the doctree schema, a standard JSON file format for describing library
2 | // documentation.
3 | //
4 | // tree-sitter is used to emit documentation in this format, and the doctree frontend renders it.
5 | package schema
6 |
7 | // LatestVersion of the doctree schema (semver.)
8 | const LatestVersion = "0.0.1"
9 |
10 | // Index is the top-most data structure in the doctree schema. It is produed by running a language
11 | // indexer over a directory, which may contain one or more libraries of code.
12 | type Index struct {
13 | // The version of the doctree schema in use. Set this to the LatestVersion constant.
14 | SchemaVersion string `json:"schemaVersion"`
15 |
16 | // Directory that was indexed (absolute path.)
17 | Directory string `json:"directory"`
18 |
19 | // GitRepository is the normalized Git repository URI. e.g. "https://github.com/golang/go" or
20 | // "git@github.com:golang/go" - the same value reported by `git config --get remote.origin.url`
21 | // with `git@github.com:foo/bar` rewritten to `git://github.com/foo/bar`, credentials removed,
22 | // any ".git" suffix removed, and any leading "/" prefix removed.
23 | //
24 | // Empty string if the indexed directory was not a Git repository.
25 | GitRepository string `json:"gitRepository"`
26 |
27 | // GitCommitID is the SHA commit hash of the Git repository revision at the time of indexing, as
28 | // reported by `git rev-parse HEAD`.
29 | //
30 | // Empty string if the indexed directory was not a Git repository.
31 | GitCommitID string `json:"gitCommitID"`
32 |
33 | // GitRefName is the current Git ref name (branch name, tag name, etc.) as reported by
34 | // `git rev-parse --abbrev-ref HEAD`
35 | //
36 | // Empty string if the indexed directory was not a Git repository.
37 | GitRefName string `json:"gitRefName"`
38 |
39 | // CreatedAt time of the index (RFC3339)
40 | CreatedAt string `json:"createdAt"`
41 |
42 | // NumFiles indexed.
43 | NumFiles int `json:"numFiles"`
44 |
45 | // NumBytes indexed.
46 | NumBytes int `json:"numBytes"`
47 |
48 | // DurationSeconds is how long indexing took.
49 | DurationSeconds float64 `json:"durationSeconds"`
50 |
51 | // Language name.
52 | Language Language `json:"language"`
53 |
54 | // Library documentation.
55 | Libraries []Library `json:"libraries"`
56 | }
57 |
58 | // Language name in canonical form, e.g. "Go", "Objective-C", etc.
59 | type Language struct {
60 | // Title of the language, e.g. "C++" or "Objective-C"
61 | Title string `json:"title"`
62 |
63 | // ID of the language, e.g. "cpp", "objc". Lowercase.
64 | ID string `json:"id"`
65 | }
66 |
67 | // Language name constants.
68 | var (
69 | LanguageC = Language{Title: "C", ID: "c"}
70 | LanguageCpp = Language{Title: "C++", ID: "cpp"}
71 | LanguageGo = Language{Title: "Go", ID: "go"}
72 | LanguageJava = Language{Title: "Java", ID: "java"}
73 | LanguageJavaScript = Language{Title: "JavaScript", ID: "javascript"}
74 | LanguageObjC = Language{Title: "Objective-C", ID: "objc"}
75 | LanguagePython = Language{Title: "Python", ID: "python"}
76 | LanguageTypeScript = Language{Title: "TypeScript", ID: "typescript"}
77 | LanguageZig = Language{Title: "Zig", ID: "zig"}
78 | LanguageMarkdown = Language{Title: "Markdown", ID: "markdown"}
79 | )
80 |
81 | // Library documentation, represents a code library / a logical unit of code typically distributed
82 | // by package managers.
83 | type Library struct {
84 | // Name of the library
85 | Name string `json:"name"`
86 |
87 | // ID of this repository. Many languages have a unique identifier, for example in Java this may
88 | // be "com.google.android.webview" in Python it may be the PyPi package name. For Rust, the
89 | // Cargo crate name, etc.
90 | ID string `json:"id"`
91 |
92 | // Version of the library
93 | Version string `json:"version"`
94 |
95 | // Version string type, e.g. "semver", "commit"
96 | VersionType string `json:"versionType"`
97 |
98 | // Pages of documentation for the library.
99 | Pages []Page `json:"pages"`
100 | }
101 |
102 | // Page is a single page of documentation, and typically gets rendered as a single page in the
103 | // browser.
104 | type Page struct {
105 | // Path of the page relative to the library. This is the URL path and does not necessarily have
106 | // to match up with filepaths.
107 | Path string `json:"path"`
108 |
109 | // Title of the page.
110 | Title string `json:"title"`
111 |
112 | // The detail
113 | Detail Markdown `json:"detail"`
114 |
115 | // SearchKey describes a single string a user would type in to a search bar to find this
116 | // page. For example, in Go this might be "net/http"
117 | //
118 | // This is a list of strings to diffentiate the different "parts" of the string, for Go it would
119 | // actually be ["net", "/", "http"]. The search engine will do fuzzy prefix/suffix matching of
120 | // each *part* of the key. For example, a query for "net" would be treated as "*net*".
121 | //
122 | // The key should aim to be unique within the scope of the directory and language that was
123 | // indexed (you can imagine the key is prefixed with the language name and directory/repository
124 | // name for you.)
125 | SearchKey []string `json:"searchKey"`
126 |
127 | // Sections on the page.
128 | Sections []Section `json:"sections"`
129 |
130 | // Subpages of this one.
131 | Subpages []Page `json:"subpages,omitempty"`
132 | }
133 |
134 | // Section represents a single section of documentation on a page. These give you building blocks
135 | // to arrange the page how you see fit. For example, in Go maybe you want documentation to be
136 | // structured as:
137 | //
138 | // * Overview
139 | // * Constants
140 | // * Variables
141 | // * Functions
142 | // * func SetURLVars
143 | // * Types
144 | // * type Route
145 | // * (r) GetName
146 | //
147 | // Each of these bullet points in the list above is a Section!
148 | type Section struct {
149 | // The ID of this section, used in the hyperlink to link to this section of the page.
150 | ID string `json:"id"`
151 |
152 | // Category indicates if this section is just describing a category of children, for example
153 | // if this section has the label "Functions" and Children are all of the functions in the
154 | // library. This information is used to pick out key sections that should be shown in high-level
155 | // navigation.
156 | Category bool `json:"category"`
157 |
158 | // ShortLabel is the shortest string that can describe this section relative to the parent. For
159 | // example, in Go this may be `(r) GetName` as a reduced form of `func (r *Route) GetName`.
160 | ShortLabel string `json:"shortLabel"`
161 |
162 | // The label of this section.
163 | Label Markdown `json:"label"`
164 |
165 | // The detail
166 | Detail Markdown `json:"detail"`
167 |
168 | // SearchKey describes a single string a user would type in to a search bar to find this
169 | // section. For example, in Go this might be "net/http.Client.PostForm"
170 | //
171 | // This is a list of strings to diffentiate the different "parts" of the string, for Go it would
172 | // actually be ["net", "/", "http", ".", "Client", ".", "PostForm"]. The search engine will do
173 | // fuzzy prefix/suffix matching of each *part* of the key. For example, a query for
174 | // "net.PostForm" would be treated as "*net*.*PostForm*".
175 | //
176 | // The key should aim to be unique within the scope of the directory and language that was
177 | // indexed (you can imagine the key is prefixed with the language name and directory/repository
178 | // name for you.)
179 | SearchKey []string `json:"searchKey"`
180 |
181 | // Any children sections. For example, if this section represents a class the children could be
182 | // the methods of the class and they would be rendered immediately below this section and
183 | // indicated as being children of the parent section.
184 | Children []Section `json:"children"`
185 | }
186 |
187 | // Markdown text.
188 | type Markdown string
189 |
--------------------------------------------------------------------------------
/doctree/sourcegraph/client.go:
--------------------------------------------------------------------------------
1 | // Package sourcegraph provides a Sourcegraph API client.
2 | package sourcegraph
3 |
4 | import (
5 | "context"
6 | "net"
7 | "net/http"
8 | "time"
9 | )
10 |
11 | // Options describes client options.
12 | type Options struct {
13 | // URL is the Sourcegraph instance URL, e.g. "https://sourcegraph.com"
14 | URL string
15 |
16 | // Token is a Sourcegraph API access token, or an empty string.
17 | Token string
18 | }
19 |
20 | type Client interface {
21 | DefRefImpl(context.Context, DefRefImplArgs) (*Repository, error)
22 | }
23 |
24 | func New(opt Options) Client {
25 | tr := &http.Transport{
26 | MaxIdleConns: 10,
27 | IdleConnTimeout: 30 * time.Second,
28 | Dial: func(network, addr string) (net.Conn, error) {
29 | return net.DialTimeout(network, addr, 3*time.Second)
30 | },
31 | ResponseHeaderTimeout: 0,
32 | }
33 | return &graphQLClient{
34 | opt: opt,
35 | client: &http.Client{
36 | Transport: tr,
37 | Timeout: 60 * time.Second,
38 | },
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/doctree/sourcegraph/graphql.go:
--------------------------------------------------------------------------------
1 | // Package sourcegraph provides the Sourcegraph API.
2 | package sourcegraph
3 |
4 | import (
5 | "bytes"
6 | "context"
7 | "encoding/json"
8 | "fmt"
9 | "io"
10 | "net/http"
11 |
12 | "github.com/pkg/errors"
13 | )
14 |
15 | // graphQLQuery describes a general GraphQL query and its variables.
16 | type graphQLQuery struct {
17 | Query string `json:"query"`
18 | Variables any `json:"variables"`
19 | }
20 |
21 | type graphQLClient struct {
22 | opt Options
23 | client *http.Client
24 | }
25 |
26 | // requestGraphQL performs a GraphQL request with the given query and variables.
27 | // search executes the given search query. The queryName is used as the source of the request.
28 | // The result will be decoded into the given pointer.
29 | func (c *graphQLClient) requestGraphQL(ctx context.Context, queryName string, query string, variables any) ([]byte, error) {
30 | var buf bytes.Buffer
31 | err := json.NewEncoder(&buf).Encode(graphQLQuery{
32 | Query: query,
33 | Variables: variables,
34 | })
35 | if err != nil {
36 | return nil, errors.Wrap(err, "Encode")
37 | }
38 |
39 | req, err := http.NewRequestWithContext(ctx, "POST", c.opt.URL+"/.api/graphql?doctree"+queryName, &buf)
40 | if err != nil {
41 | return nil, errors.Wrap(err, "Post")
42 | }
43 |
44 | if c.opt.Token != "" {
45 | req.Header.Set("Authorization", "token "+c.opt.Token)
46 | }
47 | req.Header.Set("Content-Type", "application/json")
48 |
49 | resp, err := c.client.Do(req)
50 | if err != nil {
51 | return nil, errors.Wrap(err, "Post")
52 | }
53 | defer resp.Body.Close()
54 |
55 | data, err := io.ReadAll(resp.Body)
56 | if err != nil {
57 | return nil, errors.Wrap(err, "ReadAll")
58 | }
59 |
60 | var errs struct {
61 | Errors []any
62 | }
63 | if err := json.Unmarshal(data, &errs); err != nil {
64 | return nil, errors.Wrap(err, "Unmarshal errors")
65 | }
66 | if len(errs.Errors) > 0 {
67 | return nil, fmt.Errorf("graphql error: %v", errs.Errors)
68 | }
69 | return data, nil
70 | }
71 |
--------------------------------------------------------------------------------
/doctree/sourcegraph/query_def_ref_impl.go:
--------------------------------------------------------------------------------
1 | package sourcegraph
2 |
3 | import (
4 | "context"
5 | "encoding/json"
6 | "fmt"
7 | "strings"
8 |
9 | "github.com/pkg/errors"
10 | )
11 |
12 | func defRefImplQuery(args DefRefImplArgs) string {
13 | var aliased []string
14 | var params []string
15 |
16 | for fileIndex, file := range args.Files {
17 | params = append(params, fmt.Sprintf("$file%vpath: String!", fileIndex))
18 | var aliasedFile []string
19 | for i := range file.Positions {
20 | params = append(params, fmt.Sprintf("$file%vline%v: Int!", fileIndex, i))
21 | params = append(params, fmt.Sprintf("$file%vcharacter%v: Int!", fileIndex, i))
22 | aliasedFile = append(aliasedFile, strings.ReplaceAll(strings.ReplaceAll(`
23 | references#P: references(
24 | line: $file#Fline#P
25 | character: $file#Fcharacter#P
26 | first: $firstReferences
27 | after: $afterReferences
28 | filter: $filter
29 | ) {
30 | ...LocationConnectionFields
31 | }
32 | # TODO: query implementations once that API does not take down prod:
33 | # https://github.com/sourcegraph/sourcegraph/issues/36882
34 | #implementations#P: implementations(
35 | # line: $file#Fline#P
36 | # character: $file#Fcharacter#P
37 | # first: $firstImplementations
38 | # after: $afterImplementations
39 | # filter: $filter
40 | #) {
41 | # ...LocationConnectionFields
42 | #}
43 | definitions#P: definitions(line: $file#Fline#P, character: $file#Fcharacter#P, filter: $filter) {
44 | ...LocationConnectionFields
45 | }
46 | `, "#F", fmt.Sprint(fileIndex)), "#P", fmt.Sprint(i)))
47 | }
48 | aliased = append(aliased, fmt.Sprintf(`
49 | blob%v: blob(path: $file%vpath) {
50 | lsif {
51 | %s
52 | }
53 | }
54 | `, fileIndex, fileIndex, strings.Join(aliasedFile, "\n")))
55 | }
56 |
57 | return fmt.Sprintf(`
58 | fragment LocationConnectionFields on LocationConnection {
59 | nodes {
60 | url
61 | resource {
62 | path
63 | content
64 | repository {
65 | name
66 | }
67 | commit {
68 | oid
69 | }
70 | }
71 | range {
72 | start {
73 | line
74 | character
75 | }
76 | end {
77 | line
78 | character
79 | }
80 | }
81 | }
82 | pageInfo {
83 | endCursor
84 | }
85 | }
86 |
87 | query UsePreciseCodeIntelForPosition(
88 | $repositoryCloneUrl: String!,
89 | $commit: String!,
90 | $afterReferences: String,
91 | $firstReferences: Int,
92 | # TODO: query implementations once that API does not take down prod:
93 | # https://github.com/sourcegraph/sourcegraph/issues/36882
94 | #$afterImplementations: String,
95 | #$firstImplementations: Int,
96 | $filter: String, %s) {
97 | repository(cloneURL: $repositoryCloneUrl) {
98 | id
99 | name
100 | stars
101 | isFork
102 | isArchived
103 | commit(rev: $commit) {
104 | id
105 | %s
106 | }
107 | }
108 | }
109 | `, strings.Join(params, ", "), strings.Join(aliased, "\n"))
110 | }
111 |
112 | type Position struct {
113 | Line uint `json:"line"`
114 | Character uint `json:"character"`
115 | }
116 |
117 | type File struct {
118 | Path string `json:"path"`
119 | Positions []Position
120 | }
121 |
122 | type DefRefImplArgs struct {
123 | AfterImplementations *string `json:"afterImplementations"`
124 | AfterReferences *string `json:"afterReferences"`
125 | RepositoryCloneURL string `json:"repositoryCloneUrl"`
126 | Files []File
127 | Commit string `json:"commit"`
128 | Filter *string `json:"filter"`
129 | FirstImplementations uint `json:"firstImplementations"`
130 | FirstReferences uint `json:"firstReferences"`
131 | }
132 |
133 | func (c *graphQLClient) DefRefImpl(ctx context.Context, args DefRefImplArgs) (*Repository, error) {
134 | vars := map[string]interface{}{
135 | "afterImplementations": args.AfterImplementations,
136 | "afterReferences": args.AfterReferences,
137 | "repositoryCloneUrl": args.RepositoryCloneURL,
138 | "commit": args.Commit,
139 | "filter": args.Filter,
140 | "firstImplementations": args.FirstImplementations,
141 | "firstReferences": args.FirstReferences,
142 | }
143 | for fileIndex, file := range args.Files {
144 | vars[fmt.Sprintf("file%vpath", fileIndex)] = file.Path
145 | for i, pos := range file.Positions {
146 | vars[fmt.Sprintf("file%vline%v", fileIndex, i)] = pos.Line
147 | vars[fmt.Sprintf("file%vcharacter%v", fileIndex, i)] = pos.Character
148 | }
149 | }
150 |
151 | resp, err := c.requestGraphQL(ctx, "DefRefImpl", defRefImplQuery(args), vars)
152 | if err != nil {
153 | return nil, errors.Wrap(err, "graphql")
154 | }
155 | var raw struct {
156 | Data struct {
157 | Repository DefRefImplRepository
158 | }
159 | }
160 | if err := json.Unmarshal(resp, &raw); err != nil {
161 | return nil, errors.Wrap(err, "Unmarshal")
162 | }
163 | var (
164 | r = raw.Data.Repository
165 | blobs []Blob
166 | )
167 | for fileIndex, file := range args.Files {
168 | rawBlob, ok := r.Commit[fmt.Sprintf("blob%v", fileIndex)]
169 | if !ok {
170 | continue
171 | }
172 | var info struct {
173 | LSIF map[string]json.RawMessage
174 | }
175 | if err := json.Unmarshal(rawBlob, &info); err != nil {
176 | return nil, errors.Wrap(err, "Unmarshal")
177 | }
178 |
179 | decodeLocation := func(name string, dst *[]Location) error {
180 | raw, ok := info.LSIF[name]
181 | if !ok {
182 | return nil
183 | }
184 | var result *Location
185 | if err := json.Unmarshal(raw, &result); err != nil {
186 | return errors.Wrap(err, "Unmarshal")
187 | }
188 | if result != nil {
189 | *dst = append(*dst, *result)
190 | }
191 | return nil
192 | }
193 | blob := Blob{LSIF: &LSIFBlob{}}
194 | for i := range file.Positions {
195 | if err := decodeLocation(fmt.Sprintf("references%v", i), &blob.LSIF.References); err != nil {
196 | return nil, errors.Wrap(err, "decodeLocation(references)")
197 | }
198 | // TODO: query implementations once that API does not take down prod:
199 | // https://github.com/sourcegraph/sourcegraph/issues/36882
200 | // if err := decodeLocation(fmt.Sprintf("implementations%v", i), &blob.LSIF.Implementations); err != nil {
201 | // return nil, errors.Wrap(err, "decodeLocation(implementations)")
202 | // }
203 | if err := decodeLocation(fmt.Sprintf("definitions%v", i), &blob.LSIF.Definitions); err != nil {
204 | return nil, errors.Wrap(err, "decodeLocation(definitions)")
205 | }
206 | }
207 | blobs = append(blobs, blob)
208 | }
209 | var commitID string
210 | _ = json.Unmarshal(r.Commit["id"], &commitID)
211 | var commitOID string
212 | _ = json.Unmarshal(r.Commit["oid"], &commitOID)
213 | return &Repository{
214 | ID: r.ID,
215 | Name: r.Name,
216 | Stars: r.Stars,
217 | IsFork: r.IsFork,
218 | IsArchived: r.IsArchived,
219 | Commit: &Commit{
220 | ID: commitID,
221 | OID: commitOID,
222 | Blobs: blobs,
223 | },
224 | }, nil
225 | }
226 |
227 | type DefRefImplRepository struct {
228 | ID string
229 | Name string
230 | Stars uint64
231 | IsFork bool
232 | IsArchived bool
233 | Commit map[string]json.RawMessage
234 | }
235 |
--------------------------------------------------------------------------------
/doctree/sourcegraph/types.go:
--------------------------------------------------------------------------------
1 | package sourcegraph
2 |
3 | type LocationResource struct {
4 | Path string
5 | Content string
6 | Repository Repository
7 | Commit Commit
8 | }
9 |
10 | // Zero is the first line. Zero is the first character.
11 | type LineOffset struct {
12 | Line uint64
13 | Character uint64
14 | }
15 |
16 | type Range struct {
17 | Start LineOffset
18 | End LineOffset
19 | }
20 |
21 | type LocationNode struct {
22 | URL string
23 | Resource LocationResource
24 | Range Range
25 | }
26 |
27 | type PageInfo struct {
28 | TotalCount uint64
29 | PageInfo struct {
30 | EndCursor *string
31 | HasNextPage bool
32 | }
33 | }
34 |
35 | type Location struct {
36 | Nodes []LocationNode
37 | PageInfo PageInfo
38 | }
39 |
40 | type LSIFBlob struct {
41 | References []Location
42 | Implementations []Location
43 | Definitions []Location
44 | }
45 |
46 | type Blob struct {
47 | LSIF *LSIFBlob
48 | }
49 |
50 | type Commit struct {
51 | ID string
52 | OID string
53 | Blobs []Blob
54 | }
55 |
56 | type Repository struct {
57 | ID string
58 | Name string
59 | Stars uint64
60 | IsFork bool
61 | IsArchived bool
62 | Commit *Commit `json:"commit"`
63 | }
64 |
--------------------------------------------------------------------------------
/frontend/assets.go:
--------------------------------------------------------------------------------
1 | package frontend
2 |
3 | import (
4 | "embed"
5 | "io/fs"
6 | )
7 |
8 | //go:embed public/**
9 | var public embed.FS
10 |
11 | func EmbeddedFS() fs.FS {
12 | f, err := fs.Sub(public, "public")
13 | if err != nil {
14 | panic(err)
15 | }
16 | return f
17 | }
18 |
--------------------------------------------------------------------------------
/frontend/elm.json:
--------------------------------------------------------------------------------
1 | {
2 | "type": "application",
3 | "source-directories": ["src"],
4 | "elm-version": "0.19.1",
5 | "dependencies": {
6 | "direct": {
7 | "NoRedInk/elm-json-decode-pipeline": "1.0.1",
8 | "dillonkearns/elm-markdown": "7.0.0",
9 | "elm/browser": "1.0.2",
10 | "elm/core": "1.0.5",
11 | "elm/html": "1.0.0",
12 | "elm/http": "2.0.0",
13 | "elm/json": "1.1.3",
14 | "elm/url": "1.0.0",
15 | "mdgriffith/elm-ui": "1.1.8"
16 | },
17 | "indirect": {
18 | "elm/bytes": "1.0.8",
19 | "elm/file": "1.0.5",
20 | "elm/parser": "1.1.0",
21 | "elm/regex": "1.0.0",
22 | "elm/time": "1.0.0",
23 | "elm/virtual-dom": "1.0.2",
24 | "rtfeldman/elm-hex": "1.0.0"
25 | }
26 | },
27 | "test-dependencies": {
28 | "direct": {},
29 | "indirect": {}
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/frontend/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "doctree-frontend",
3 | "version": "1.0.0",
4 | "description": "Frontend for Doctree",
5 | "license": "Apache",
6 | "private": true,
7 | "devDependencies": {
8 | "elm": "^0.19.1-5"
9 | },
10 | "dependencies": {
11 | "elm-live": "^4.0.2"
12 | }
13 | }
14 |
--------------------------------------------------------------------------------
/frontend/public/android-chrome-192x192.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sourcegraph/doctree/418506c648ded380cad7883dbdbb057e616febf9/frontend/public/android-chrome-192x192.png
--------------------------------------------------------------------------------
/frontend/public/android-chrome-512x512.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sourcegraph/doctree/418506c648ded380cad7883dbdbb057e616febf9/frontend/public/android-chrome-512x512.png
--------------------------------------------------------------------------------
/frontend/public/apple-touch-icon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sourcegraph/doctree/418506c648ded380cad7883dbdbb057e616febf9/frontend/public/apple-touch-icon.png
--------------------------------------------------------------------------------
/frontend/public/browserconfig.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | #00aba9
7 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/frontend/public/favicon-16x16.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sourcegraph/doctree/418506c648ded380cad7883dbdbb057e616febf9/frontend/public/favicon-16x16.png
--------------------------------------------------------------------------------
/frontend/public/favicon-32x32.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sourcegraph/doctree/418506c648ded380cad7883dbdbb057e616febf9/frontend/public/favicon-32x32.png
--------------------------------------------------------------------------------
/frontend/public/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sourcegraph/doctree/418506c648ded380cad7883dbdbb057e616febf9/frontend/public/favicon.ico
--------------------------------------------------------------------------------
/frontend/public/index-cloud.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 | doctree - 100% open source library documentation tool
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
40 |
41 |
42 |
43 |
52 |
64 |
65 |
66 |
67 |
68 |
--------------------------------------------------------------------------------
/frontend/public/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 | doctree - 100% open source library documentation tool
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
42 |
54 |
55 |
56 |
57 |
58 |
--------------------------------------------------------------------------------
/frontend/public/mstile-150x150.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sourcegraph/doctree/418506c648ded380cad7883dbdbb057e616febf9/frontend/public/mstile-150x150.png
--------------------------------------------------------------------------------
/frontend/public/opendata.js:
--------------------------------------------------------------------------------
1 | /* Plausible (https://plausible.io/), "Simple and privacy-friendly Google Analytics alternative"
2 | This is ONLY used on doctree.org, never on self-hosted/local instances (which should never
3 | contact public internet without permission.) We just use it to see if people are actually using
4 | doctree.org and if we should continue developing it.
5 |
6 | You can view the same metrics we do, publicly, here: https://plausible.io/doctree.org
7 | */
8 | !function(r,i){"use strict";var e,o=r.location,s=r.document,t=s.querySelector('[src*="'+i+'"]'),l=t&&t.getAttribute("data-domain"),p=r.localStorage.plausible_ignore;function c(e){console.warn("Ignoring Event: "+e)}function a(e,t){if(/^localhost$|^127(?:\.[0-9]+){0,2}\.[0-9]+$|^(?:0*\:)*?:?0*1$/.test(o.hostname)||"file:"===o.protocol)return c("localhost");if(!(r.phantom||r._phantom||r.__nightmare||r.navigator.webdriver||r.Cypress)){if("true"==p)return c("localStorage flag");var a={};a.n=e,a.u=o.href,a.d=l,a.r=s.referrer||null,a.w=r.innerWidth,t&&t.meta&&(a.m=JSON.stringify(t.meta)),t&&t.props&&(a.p=JSON.stringify(t.props));var n=new XMLHttpRequest;n.open("POST",i+"/api/event",!0),n.setRequestHeader("Content-Type","text/plain"),n.send(JSON.stringify(a)),n.onreadystatechange=function(){4==n.readyState&&t&&t.callback&&t.callback()}}}function n(){e!==o.pathname&&(e=o.pathname,a("pageview"))}try{var u,h=r.history;h.pushState&&(u=h.pushState,h.pushState=function(){u.apply(this,arguments),n()},r.addEventListener("popstate",n));var g=r.plausible&&r.plausible.q||[];r.plausible=a;for(var f=0;f
2 |
4 |
78 |
--------------------------------------------------------------------------------
/frontend/public/site.webmanifest:
--------------------------------------------------------------------------------
1 | {
2 | "name": "doctree",
3 | "short_name": "doctree",
4 | "icons": [
5 | {
6 | "src": "/android-chrome-192x192.png",
7 | "sizes": "192x192",
8 | "type": "image/png"
9 | },
10 | {
11 | "src": "/android-chrome-512x512.png",
12 | "sizes": "512x512",
13 | "type": "image/png"
14 | }
15 | ],
16 | "theme_color": "#ffffff",
17 | "background_color": "#ffffff",
18 | "display": "standalone"
19 | }
20 |
--------------------------------------------------------------------------------
/frontend/public/stylesheet.css:
--------------------------------------------------------------------------------
1 | .markdown {
2 | width: 100% !important;
3 | }
4 | .markdown, .markdown p {
5 | font-family: Verdana, Geneva, sans-serif;
6 | font-size: 18px;
7 | line-height: 1.25;
8 | white-space: normal;
9 | }
10 | .markdown pre {
11 | width: 100%;
12 | white-space: pre-wrap;
13 | padding: 1rem;
14 | }
15 | .markdown code {
16 | padding: 0.15rem;
17 | margin: 0.1rem;
18 | display: inline-block;
19 | }
20 | .markdown code, .markdown pre {
21 | font-family: "JetBrains Mono", monospace;
22 | font-size: 16px;
23 | background: #e6e3e3;
24 | }
25 | .markdown pre>code {
26 | background: none;
27 | padding: 0;
28 | }
29 | .markdown p + p, .markdown table + p {
30 | margin-top: 1rem;
31 | }
32 | .markdown p {
33 | margin: 0;
34 | }
35 | .markdown img {
36 | max-width: 100%;
37 | }
38 | .markdown table {
39 | margin-top: 1rem;
40 | border-collapse: collapse;
41 | width: calc(100% - 2rem - 2rem);
42 | margin-left: 2rem;
43 | }
44 | .markdown thead {
45 | border-bottom: 1px solid gray;
46 | }
47 | .markdown td, th {
48 | padding: 0.5rem;
49 | }
50 | .markdown tr:nth-child(even) {
51 | background: #f0f0f0;
52 | }
53 | .markdown th {
54 | padding-top: 1rem;
55 | padding-bottom: 1rem;
56 | text-align: left;
57 | }
58 | .markdown table code {
59 | background: 0;
60 | padding: 0;
61 | }
62 | .markdown li ul {
63 | margin-top: 1rem;
64 | margin-bottom: 1rem;
65 | }
66 | .markdown li li > * {
67 | padding-left: 1rem;
68 | }
69 | .markdown li li > *:first-child {
70 | padding-left: 0;
71 | }
72 | .markdown li li > *:last-child {
73 | margin-bottom: 1rem;
74 | }
75 | .markdown h5, h6 {
76 | font-size: 12px;
77 | }
78 | .markdown blockquote {
79 | margin: 1rem;
80 | margin-left: 0.5rem;
81 | padding: 0.5rem;
82 | border-left: 3px solid #d2d2d2;
83 | }
84 | .markdown hr {
85 | height: 2px;
86 | background: #b7b7b7;
87 | border: 0;
88 | width: 100%;
89 | }
90 |
--------------------------------------------------------------------------------
/frontend/src/API.elm:
--------------------------------------------------------------------------------
1 | module API exposing (..)
2 |
3 | import APISchema
4 | import Http
5 | import Json.Decode
6 | import Url.Builder
7 | import Util
8 |
9 |
10 | fetchProjectList : (Result Http.Error (List String) -> msg) -> Cmd msg
11 | fetchProjectList msg =
12 | Http.get
13 | { url = "/api/list"
14 | , expect = Http.expectJson msg (Json.Decode.list Json.Decode.string)
15 | }
16 |
17 |
18 | fetchProject : (Result Http.Error APISchema.ProjectIndexes -> msg) -> String -> Cmd msg
19 | fetchProject msg projectName =
20 | Http.get
21 | { url = Url.Builder.absolute [ "api", "get" ] [ Url.Builder.string "name" projectName ]
22 | , expect = Http.expectJson msg APISchema.projectIndexesDecoder
23 | }
24 |
25 |
26 | fetchSearchResults :
27 | (Result Http.Error APISchema.SearchResults -> msg)
28 | -> String
29 | -> Bool
30 | -> Maybe String
31 | -> Cmd msg
32 | fetchSearchResults msg query intent projectName =
33 | Http.get
34 | { url =
35 | Url.Builder.absolute [ "api", "search" ]
36 | [ Url.Builder.string "query" query
37 | , Url.Builder.string "autocomplete" (Util.boolToString (intent == False))
38 | , Url.Builder.string "project" (Maybe.withDefault "" projectName)
39 | ]
40 | , expect = Http.expectJson msg APISchema.searchResultsDecoder
41 | }
42 |
43 |
44 | type alias PageID =
45 | { projectName : String
46 | , language : String
47 | , pagePath : String
48 | }
49 |
50 |
51 | fetchPage : (Result Http.Error APISchema.Page -> msg) -> PageID -> Cmd msg
52 | fetchPage msg pageID =
53 | Http.get
54 | { url =
55 | Url.Builder.absolute [ "api", "get-page" ]
56 | [ Url.Builder.string "project" pageID.projectName
57 | , Url.Builder.string "language" pageID.language
58 | , Url.Builder.string "page" pageID.pagePath
59 | ]
60 | , expect = Http.expectJson msg APISchema.pageDecoder
61 | }
62 |
--------------------------------------------------------------------------------
/frontend/src/APISchema.elm:
--------------------------------------------------------------------------------
1 | module APISchema exposing (..)
2 |
3 | import Dict exposing (Dict)
4 | import Json.Decode as Decode exposing (Decoder)
5 | import Json.Decode.Pipeline as Pipeline
6 | import Schema
7 |
8 |
9 |
10 | -- decoder for: /api/list
11 |
12 |
13 | type alias ProjectList =
14 | List String
15 |
16 |
17 | projectListDecoder : Decoder ProjectList
18 | projectListDecoder =
19 | Decode.list Decode.string
20 |
21 |
22 |
23 | -- decoder for: /api/get-page?project=github.com/sourcegraph/sourcegraph&language=markdown&page=/README.md
24 |
25 |
26 | type alias Page =
27 | Schema.Page
28 |
29 |
30 | pageDecoder =
31 | Schema.pageDecoder
32 |
33 |
34 |
35 | -- decoder for: /api/get?name=github.com/sourcegraph/sourcegraph
36 | -- decoder for: /api/get-index?name=github.com/sourcegraph/sourcegraph
37 |
38 |
39 | type alias ProjectIndexes =
40 | Dict String Schema.Index
41 |
42 |
43 | projectIndexesDecoder : Decoder ProjectIndexes
44 | projectIndexesDecoder =
45 | Decode.dict Schema.indexDecoder
46 |
47 |
48 |
49 | -- decoder for: /api/search?query=foobar
50 |
51 |
52 | type alias SearchResults =
53 | List SearchResult
54 |
55 |
56 | searchResultsDecoder : Decoder SearchResults
57 | searchResultsDecoder =
58 | Decode.list searchResultDecoder
59 |
60 |
61 | searchResultDecoder : Decoder SearchResult
62 | searchResultDecoder =
63 | Decode.succeed SearchResult
64 | |> Pipeline.required "language" Decode.string
65 | |> Pipeline.required "projectName" Decode.string
66 | |> Pipeline.required "searchKey" Decode.string
67 | |> Pipeline.required "path" Decode.string
68 | |> Pipeline.required "id" Decode.string
69 | |> Pipeline.required "score" Decode.float
70 |
71 |
72 | type alias SearchResult =
73 | { language : String
74 | , projectName : String
75 | , searchKey : String
76 | , path : String
77 | , id : String
78 | , score : Float
79 | }
80 |
--------------------------------------------------------------------------------
/frontend/src/Flags.elm:
--------------------------------------------------------------------------------
1 | module Flags exposing (Decoded, Flags, decode)
2 |
3 | import Json.Decode as Decode
4 | import Json.Decode.Pipeline as Pipeline
5 |
6 |
7 | type alias Flags =
8 | Decode.Value
9 |
10 |
11 | decode : Flags -> Decoded
12 | decode flags =
13 | Decode.decodeValue decoder flags
14 | |> Result.withDefault { cloudMode = False }
15 |
16 |
17 | decoder : Decode.Decoder Decoded
18 | decoder =
19 | Decode.succeed Decoded
20 | |> Pipeline.required "cloudMode" Decode.bool
21 |
22 |
23 | type alias Decoded =
24 | { cloudMode : Bool
25 | }
26 |
--------------------------------------------------------------------------------
/frontend/src/Home.elm:
--------------------------------------------------------------------------------
1 | module Home exposing (Model, Msg(..), view)
2 |
3 | import Browser
4 | import Element as E
5 | import Element.Font as Font
6 | import Element.Lazy
7 | import Http
8 | import Search
9 | import Style
10 | import Util exposing (httpErrorToString)
11 |
12 |
13 | type alias Model =
14 | { projectList : Maybe (Result Http.Error (List String))
15 | , search : Search.Model
16 | }
17 |
18 |
19 | type Msg
20 | = SearchMsg Search.Msg
21 |
22 |
23 | view : Bool -> Model -> Browser.Document Msg
24 | view cloudMode model =
25 | { title = "doctree"
26 | , body =
27 | [ E.layout (List.concat [ Style.layout, [ E.width E.fill ] ])
28 | (case model.projectList of
29 | Just response ->
30 | case response of
31 | Ok list ->
32 | E.column [ E.centerX, E.width (E.fill |> E.maximum 700), E.paddingXY 0 64 ]
33 | [ logo
34 | , E.el
35 | [ E.paddingEach { top = 64, right = 0, bottom = 0, left = 0 }
36 | , E.width E.fill
37 | ]
38 | (E.map (\v -> SearchMsg v) Search.searchInput)
39 | , if model.search.query /= "" then
40 | Element.Lazy.lazy
41 | (\results -> E.map (\v -> SearchMsg v) (Search.searchResults results))
42 | model.search.results
43 |
44 | else if cloudMode then
45 | E.column [ E.centerX ]
46 | [ Style.h2 [ E.paddingXY 0 32 ] (E.text "# Try doctree")
47 | , E.column [] (projectsList list)
48 | , Style.h2 [ E.paddingEach { top = 32, right = 0, bottom = 0, left = 0 } ]
49 | (E.text "# Add your repository to doctree.org")
50 | , Style.paragraph [ E.paddingEach { top = 32, right = 0, bottom = 0, left = 0 } ]
51 | [ E.text "Simply visit e.g. "
52 | , E.el [ Font.underline ] (E.text "https://doctree.org/github.com/my/repo")
53 | , E.text " (replace my/repo in the URL with your repository) and the server will clone & index your repository."
54 | ]
55 | , Style.h2 [ E.paddingEach { top = 32, right = 0, bottom = 0, left = 0 } ]
56 | (E.text "# About doctree")
57 | , Style.h3 [ E.paddingEach { top = 32, right = 0, bottom = 0, left = 0 } ]
58 | (E.text "# 100% open-source library docs tool for every language")
59 | , Style.paragraph [ E.paddingEach { top = 16, right = 0, bottom = 0, left = 0 } ]
60 | [ E.text "Available "
61 | , E.link [ Font.underline ] { url = "https://github.com/sourcegraph/doctree", label = E.text "on GitHub" }
62 | , E.text ", doctree provides first-class library documentation for every language (based on tree-sitter), with symbol search & more. If connected to Sourcegraph, it can automatically surface real-world usage examples."
63 | ]
64 | , Style.h3 [ E.paddingEach { top = 32, right = 0, bottom = 0, left = 0 } ] (E.text "# Run locally, self-host, or use doctree.org")
65 | , Style.paragraph [ E.paddingEach { top = 16, right = 0, bottom = 0, left = 0 } ]
66 | [ E.text "doctree is a single binary, lightweight, and designed to run on your local machine. It can be self-hosted, and used via doctree.org with any GitHub repository. "
67 | , E.link [ Font.underline ] { url = "https://github.com/sourcegraph/doctree#installation", label = E.text "installation instructions" }
68 | ]
69 | , Style.h3 [ E.paddingEach { top = 32, right = 0, bottom = 0, left = 0 } ]
70 | (E.text "# Experimental! Early stages!")
71 | , Style.paragraph [ E.paddingEach { top = 16, right = 0, bottom = 0, left = 0 } ]
72 | [ E.text "Extremely early stages, we're working on adding more languages, polishing the experience, and adding usage examples. It's all very early and not yet ready for production use, please bear with us!"
73 | ]
74 | , Style.paragraph [ E.paddingEach { top = 16, right = 0, bottom = 0, left = 0 } ]
75 | [ E.text "Please see "
76 | , E.link [ Font.underline ] { url = "https://github.com/sourcegraph/doctree/issues/27", label = E.text "the v1.0 roadmap" }
77 | , E.text " for more, ideas welcome!"
78 | ]
79 | , Style.h3 [ E.paddingEach { top = 32, right = 0, bottom = 0, left = 0 } ]
80 | (E.text "# Join us on Discord")
81 | , Style.paragraph [ E.paddingEach { top = 16, right = 0, bottom = 0, left = 0 } ]
82 | [ E.text "If you think what we're building is a good idea, we'd love to hear your thoughts! "
83 | , E.link [ Font.underline ] { url = "https://discord.gg/vqsBW8m5Y8", label = E.text "Discord invite" }
84 | ]
85 | , Style.h3 [ E.paddingEach { top = 32, right = 0, bottom = 0, left = 0 } ]
86 | (E.text "# Language support")
87 | , Style.paragraph [ E.paddingEach { top = 16, right = 0, bottom = 0, left = 0 } ]
88 | [ E.text "Adding support for more languages is easy. To request support for a language "
89 | , E.link [ Font.underline ] { url = "https://github.com/sourcegraph/doctree/issues/10", label = E.text "comment on this issue" }
90 | , E.text "!"
91 | ]
92 | , E.table [ E.paddingEach { top = 16, right = 0, bottom = 0, left = 0 } ]
93 | { data = supportedLanguages
94 | , columns =
95 | [ { header = Style.tableHeader (E.text "language")
96 | , width = E.fill
97 | , view = \lang -> Style.tableCell (E.text lang.name)
98 | }
99 | , { header = Style.tableHeader (E.text "functions")
100 | , width = E.fill
101 | , view = \lang -> Style.tableCell (E.text lang.functions)
102 | }
103 | , { header = Style.tableHeader (E.text "types")
104 | , width = E.fill
105 | , view = \lang -> Style.tableCell (E.text lang.types)
106 | }
107 | , { header = Style.tableHeader (E.text "methods")
108 | , width = E.fill
109 | , view = \lang -> Style.tableCell (E.text lang.methods)
110 | }
111 | , { header = Style.tableHeader (E.text "consts/vars")
112 | , width = E.fill
113 | , view = \lang -> Style.tableCell (E.text lang.constsVars)
114 | }
115 | , { header = Style.tableHeader (E.text "search")
116 | , width = E.fill
117 | , view = \lang -> Style.tableCell (E.text lang.search)
118 | }
119 | , { header = Style.tableHeader (E.text "usage examples")
120 | , width = E.fill
121 | , view = \lang -> Style.tableCell (E.text lang.usageExamples)
122 | }
123 | , { header = Style.tableHeader (E.text "code intel")
124 | , width = E.fill
125 | , view = \lang -> Style.tableCell (E.text lang.codeIntel)
126 | }
127 | ]
128 | }
129 | ]
130 |
131 | else
132 | E.column [ E.centerX ]
133 | [ Style.h2 [ E.paddingXY 0 32 ] (E.text "# Your projects")
134 | , E.column [] (projectsList list)
135 | , Style.h2 [ E.paddingXY 0 32 ] (E.text "# Index a project")
136 | , E.row [ Font.size 16 ]
137 | [ E.text "$ "
138 | , E.text "doctree index ."
139 | ]
140 | ]
141 | ]
142 |
143 | Err err ->
144 | E.text (httpErrorToString err)
145 |
146 | Nothing ->
147 | E.text "loading.."
148 | )
149 | ]
150 | }
151 |
152 |
153 | type alias SupportedLanguage =
154 | { name : String
155 | , functions : String
156 | , types : String
157 | , methods : String
158 | , constsVars : String
159 | , search : String
160 | , usageExamples : String
161 | , codeIntel : String
162 | }
163 |
164 |
165 | supportedLanguages : List SupportedLanguage
166 | supportedLanguages =
167 | [ { name = "Go"
168 | , functions = "✅"
169 | , types = "✅"
170 | , methods = "❌"
171 | , constsVars = "❌"
172 | , search = "✅"
173 | , usageExamples = "❌"
174 | , codeIntel = "❌"
175 | }
176 | , { name = "Python"
177 | , functions = "✅"
178 | , types = "❌"
179 | , methods = "❌"
180 | , constsVars = "❌"
181 | , search = "✅"
182 | , usageExamples = "❌"
183 | , codeIntel = "❌"
184 | }
185 | , { name = "Zig"
186 | , functions = "✅"
187 | , types = "❌"
188 | , methods = "partial"
189 | , constsVars = "❌"
190 | , search = "✅"
191 | , usageExamples = "❌"
192 | , codeIntel = "❌"
193 | }
194 | , { name = "Markdown"
195 | , functions = "n/a"
196 | , types = "❌"
197 | , methods = "n/a"
198 | , constsVars = "❌"
199 | , search = "✅"
200 | , usageExamples = "❌"
201 | , codeIntel = "❌"
202 | }
203 | ]
204 |
205 |
206 | logo =
207 | E.row [ E.centerX ]
208 | [ E.image
209 | [ E.width (E.px 120)
210 | , E.paddingEach { top = 0, right = 140, bottom = 0, left = 0 }
211 | ]
212 | { src = "/mascot.svg", description = "cute computer / doctree mascot" }
213 | , E.column []
214 | [ E.el [ Font.size 16, Font.bold, E.alignRight ] (E.text "v0.1")
215 | , E.el [ Font.size 64, Font.bold ] (E.text "doctree")
216 | , E.el [ Font.semiBold ] (E.text "documentation for every language")
217 | ]
218 | ]
219 |
220 |
221 | projectsList list =
222 | List.map
223 | (\projectName ->
224 | E.link [ E.paddingXY 0 4 ]
225 | { url = projectName
226 | , label =
227 | E.row []
228 | [ E.text "• "
229 | , E.el [ Font.underline ] (E.text projectName)
230 | ]
231 | }
232 | )
233 | list
234 |
--------------------------------------------------------------------------------
/frontend/src/Main.elm:
--------------------------------------------------------------------------------
1 | module Main exposing (Model, Msg(..), init, main, subscriptions, update, view)
2 |
3 | import API
4 | import APISchema
5 | import Browser
6 | import Browser.Navigation as Nav
7 | import Flags exposing (Flags)
8 | import Home
9 | import Html exposing (..)
10 | import Html.Attributes exposing (..)
11 | import Http
12 | import Project exposing (Msg(..))
13 | import Route exposing (Route(..), toRoute)
14 | import Search
15 | import Url
16 | import Util
17 |
18 |
19 | main : Program Flags Model Msg
20 | main =
21 | Browser.application
22 | { init = init
23 | , view = view
24 | , update = update
25 | , subscriptions = subscriptions
26 | , onUrlChange = UrlChanged
27 | , onUrlRequest = LinkClicked
28 | }
29 |
30 |
31 | type alias Model =
32 | { flags : Flags.Decoded
33 | , key : Nav.Key
34 | , url : Url.Url
35 | , route : Route
36 | , projectList : Maybe (Result Http.Error (List String))
37 | , search : Search.Model
38 | , currentProjectName : Maybe String
39 | , projectIndexes : Maybe (Result Http.Error APISchema.ProjectIndexes)
40 | , projectPage : Maybe Project.Model
41 | }
42 |
43 |
44 | init : Flags -> Url.Url -> Nav.Key -> ( Model, Cmd Msg )
45 | init flags url key =
46 | let
47 | ( searchModel, searchCmd ) =
48 | Search.init Nothing
49 |
50 | route =
51 | toRoute (Url.toString url)
52 |
53 | projectPage =
54 | Maybe.map (\v -> Project.init v) (Project.fromRoute route)
55 | in
56 | ( { flags = Flags.decode flags
57 | , key = key
58 | , url = url
59 | , route = route
60 | , projectList = Nothing
61 | , search = searchModel
62 | , currentProjectName = Nothing
63 | , projectIndexes = Nothing
64 | , projectPage = Maybe.map (\( subModel, _ ) -> subModel) projectPage
65 | }
66 | , Cmd.batch
67 | [ case route of
68 | Route.Home _ ->
69 | API.fetchProjectList GotProjectList
70 |
71 | _ ->
72 | Cmd.none
73 | , Cmd.map (\v -> SearchMsg v) searchCmd
74 | , case projectPage of
75 | Just ( _, subCmds ) ->
76 | Cmd.map (\msg -> ProjectPage msg) subCmds
77 |
78 | Nothing ->
79 | Cmd.none
80 | ]
81 | )
82 |
83 |
84 | type Msg
85 | = NoOp
86 | | ReplaceUrlSilently String
87 | | LinkClicked Browser.UrlRequest
88 | | UrlChanged Url.Url
89 | | GotProjectList (Result Http.Error (List String))
90 | | SearchMsg Search.Msg
91 | | GetProject String
92 | | GotProject (Result Http.Error APISchema.ProjectIndexes)
93 | | ProjectPage Project.Msg
94 | | ProjectPageUpdate Project.UpdateMsg
95 |
96 |
97 | update : Msg -> Model -> ( Model, Cmd Msg )
98 | update msg model =
99 | case msg of
100 | NoOp ->
101 | ( model, Cmd.none )
102 |
103 | ReplaceUrlSilently newAbsoluteUrl ->
104 | let
105 | prefix =
106 | Url.toString
107 | { protocol = model.url.protocol
108 | , host = model.url.host
109 | , port_ = model.url.port_
110 | , path = ""
111 | , query = Nothing
112 | , fragment = Nothing
113 | }
114 |
115 | newUrl =
116 | Url.fromString (String.concat [ prefix, newAbsoluteUrl ])
117 | in
118 | case newUrl of
119 | Just url ->
120 | -- Note: By updating URL here, we ensure UrlChanged msg does nothing
121 | ( { model
122 | | url = url
123 | , route = toRoute (Url.toString url)
124 | }
125 | , Nav.replaceUrl model.key newAbsoluteUrl
126 | )
127 |
128 | Nothing ->
129 | ( model, Cmd.none )
130 |
131 | LinkClicked urlRequest ->
132 | case urlRequest of
133 | Browser.Internal url ->
134 | ( model, Nav.pushUrl model.key (Url.toString url) )
135 |
136 | Browser.External href ->
137 | ( model, Nav.load href )
138 |
139 | UrlChanged url ->
140 | let
141 | route =
142 | toRoute (Url.toString url)
143 | in
144 | if url.path /= model.url.path then
145 | -- Page changed
146 | let
147 | projectPage =
148 | Maybe.map (\v -> Project.init v) (Project.fromRoute route)
149 | in
150 | ( { model
151 | | url = url
152 | , route = route
153 | , projectPage = Maybe.map (\( subModel, _ ) -> subModel) projectPage
154 | }
155 | , Cmd.batch
156 | [ case route of
157 | Route.Home _ ->
158 | API.fetchProjectList GotProjectList
159 |
160 | _ ->
161 | Cmd.none
162 | , case projectPage of
163 | Just ( _, subCmds ) ->
164 | Cmd.map (\m -> ProjectPage m) subCmds
165 |
166 | Nothing ->
167 | Cmd.none
168 | ]
169 | )
170 |
171 | else
172 | case route of
173 | Route.ProjectLanguagePage _ _ _ newSectionID _ ->
174 | if equalExceptSectionID model.route route then
175 | -- Only the section ID changed.
176 | update
177 | (ProjectPageUpdate
178 | (Project.NavigateToSectionID newSectionID)
179 | )
180 | { model | url = url, route = route }
181 |
182 | else
183 | ( { model | url = url, route = route }
184 | , Cmd.none
185 | )
186 |
187 | _ ->
188 | ( { model | url = url, route = route }
189 | , Cmd.none
190 | )
191 |
192 | GotProjectList projectList ->
193 | ( { model | projectList = Just projectList }, Cmd.none )
194 |
195 | SearchMsg searchMsg ->
196 | let
197 | ( searchModel, searchCmd ) =
198 | Search.update searchMsg model.search
199 | in
200 | ( { model | search = searchModel }
201 | , Cmd.map (\v -> SearchMsg v) searchCmd
202 | )
203 |
204 | GetProject projectName ->
205 | Maybe.withDefault
206 | -- No project loaded yet, request it.
207 | ( { model | currentProjectName = Just projectName }
208 | , API.fetchProject GotProject projectName
209 | )
210 | -- Loaded already
211 | (model.currentProjectName
212 | |> Maybe.andThen (Util.maybeEquals projectName)
213 | |> Maybe.andThen (\_ -> model.projectIndexes)
214 | |> Maybe.map (\_ -> ( model, Cmd.none ))
215 | )
216 |
217 | GotProject result ->
218 | ( { model | projectIndexes = Just result }, Cmd.none )
219 |
220 | ProjectPage m ->
221 | update (mapProjectMsg m) model
222 |
223 | ProjectPageUpdate subMsg ->
224 | case model.projectPage of
225 | Just oldSubModel ->
226 | let
227 | ( subModel, subCmds ) =
228 | Project.update subMsg oldSubModel
229 | in
230 | ( { model | projectPage = Just subModel }, Cmd.map (\m -> ProjectPage m) subCmds )
231 |
232 | Nothing ->
233 | ( model, Cmd.none )
234 |
235 |
236 | subscriptions : Model -> Sub Msg
237 | subscriptions model =
238 | case model.projectPage of
239 | Just subModel ->
240 | Sub.map (\m -> ProjectPage m) (Project.subscriptions subModel)
241 |
242 | Nothing ->
243 | Sub.none
244 |
245 |
246 | view : Model -> Browser.Document Msg
247 | view model =
248 | case model.route of
249 | -- TODO: use search query param
250 | Route.Home _ ->
251 | let
252 | page =
253 | Home.view model.flags.cloudMode
254 | { projectList = model.projectList
255 | , search = model.search
256 | }
257 | in
258 | { title = page.title
259 | , body =
260 | List.map
261 | (\v ->
262 | Html.map
263 | (\msg ->
264 | case msg of
265 | Home.SearchMsg m ->
266 | SearchMsg m
267 | )
268 | v
269 | )
270 | page.body
271 | }
272 |
273 | Route.Project projectName searchQuery ->
274 | case model.projectPage of
275 | Just subModel ->
276 | let
277 | page =
278 | Project.viewProject model.search
279 | model.projectIndexes
280 | projectName
281 | searchQuery
282 | model.flags.cloudMode
283 | subModel
284 | in
285 | { title = page.title
286 | , body = List.map (\v -> Html.map mapProjectMsg v) page.body
287 | }
288 |
289 | Nothing ->
290 | { title = "doctree"
291 | , body =
292 | [ text "error: view Route.Project when model empty!" ]
293 | }
294 |
295 | Route.ProjectLanguage projectName language searchQuery ->
296 | case model.projectPage of
297 | Just subModel ->
298 | let
299 | page =
300 | Project.viewProjectLanguage model.search
301 | model.projectIndexes
302 | projectName
303 | language
304 | searchQuery
305 | model.flags.cloudMode
306 | subModel
307 | in
308 | { title = page.title
309 | , body = List.map (\v -> Html.map mapProjectMsg v) page.body
310 | }
311 |
312 | Nothing ->
313 | { title = "doctree"
314 | , body =
315 | [ text "error: view Route.Project when model empty!" ]
316 | }
317 |
318 | Route.ProjectLanguagePage projectName language pagePath sectionID searchQuery ->
319 | case model.projectPage of
320 | Just subModel ->
321 | let
322 | page =
323 | Project.viewProjectLanguagePage model.search
324 | projectName
325 | language
326 | pagePath
327 | sectionID
328 | searchQuery
329 | subModel
330 | in
331 | { title = page.title
332 | , body = List.map (\v -> Html.map mapProjectMsg v) page.body
333 | }
334 |
335 | Nothing ->
336 | { title = "doctree"
337 | , body =
338 | [ text "error: view Route.Project when model empty!" ]
339 | }
340 |
341 | Route.NotFound ->
342 | { title = "doctree"
343 | , body = [ text "not found" ]
344 | }
345 |
346 |
347 | mapProjectMsg : Project.Msg -> Msg
348 | mapProjectMsg msg =
349 | case msg of
350 | Project.NoOp ->
351 | NoOp
352 |
353 | Project.SearchMsg m ->
354 | SearchMsg m
355 |
356 | Project.GetProject projectName ->
357 | GetProject projectName
358 |
359 | Project.GotPage page ->
360 | ProjectPageUpdate (Project.UpdateGotPage page)
361 |
362 | Project.ObservePage ->
363 | ProjectPageUpdate Project.UpdateObservePage
364 |
365 | Project.OnObserved result ->
366 | ProjectPageUpdate (Project.UpdateOnObserved result)
367 |
368 | Project.ScrollIntoViewLater id ->
369 | ProjectPageUpdate (Project.UpdateScrollIntoViewLater id)
370 |
371 | Project.ReplaceUrlSilently newUrl ->
372 | ReplaceUrlSilently newUrl
373 |
374 |
375 | {-| Whether or not the two ProjectLanguagePage routes are equal
376 | except for the sectionID query parameter
377 | -}
378 | equalExceptSectionID : Route -> Route -> Bool
379 | equalExceptSectionID a b =
380 | case a of
381 | Route.ProjectLanguagePage projectName language pagePath sectionID searchQuery ->
382 | case b of
383 | Route.ProjectLanguagePage newProjectName newLanguage newPagePath newSectionID newSearchQuery ->
384 | projectName
385 | == newProjectName
386 | && language
387 | == newLanguage
388 | && pagePath
389 | == newPagePath
390 | && sectionID
391 | /= newSectionID
392 | && searchQuery
393 | == newSearchQuery
394 |
395 | _ ->
396 | False
397 |
398 | _ ->
399 | False
400 |
--------------------------------------------------------------------------------
/frontend/src/Markdown.elm:
--------------------------------------------------------------------------------
1 | module Markdown exposing (..)
2 |
3 | {-| This renders `Html` in an attempt to be as close as possible to
4 | the HTML output in .
5 | -}
6 |
7 | import Element as E
8 | import Html.Attributes
9 | import Markdown.Parser as Markdown
10 | import Markdown.Renderer
11 |
12 |
13 | render markdown =
14 | if False then
15 | -- DEBUG: Render Markdown as plain text for debugging.
16 | E.textColumn []
17 | (List.map
18 | (\paragraph -> E.paragraph [ E.paddingXY 0 4 ] [ E.text paragraph ])
19 | (String.split "\n" markdown)
20 | )
21 |
22 | else
23 | case
24 | markdown
25 | |> Markdown.parse
26 | |> Result.mapError deadEndsToString
27 | |> Result.andThen (\ast -> Markdown.Renderer.render Markdown.Renderer.defaultHtmlRenderer ast)
28 | of
29 | Ok rendered ->
30 | E.column [ E.htmlAttribute (Html.Attributes.class "markdown") ] (List.map (\e -> E.html e) rendered)
31 |
32 | Err errors ->
33 | E.text errors
34 |
35 |
36 | deadEndsToString deadEnds =
37 | deadEnds
38 | |> List.map Markdown.deadEndToString
39 | |> String.join "\n"
40 |
--------------------------------------------------------------------------------
/frontend/src/Ports.elm:
--------------------------------------------------------------------------------
1 | port module Ports exposing (..)
2 |
3 | import Json.Decode as Decode exposing (Decoder)
4 | import Json.Decode.Pipeline as Pipeline
5 |
6 |
7 | port observeElementID : String -> Cmd msg
8 |
9 |
10 | port onObserved : (Decode.Value -> msg) -> Sub msg
11 |
12 |
13 | observeEventsDecoder : Decoder (List ObserveEvent)
14 | observeEventsDecoder =
15 | Decode.list observeEventDecoder
16 |
17 |
18 | type alias ObserveEvent =
19 | { isIntersecting : Bool
20 | , intersectionRatio : Float
21 | , distanceToCenter : Float
22 | , targetID : String
23 | }
24 |
25 |
26 | observeEventDecoder : Decoder ObserveEvent
27 | observeEventDecoder =
28 | Decode.succeed ObserveEvent
29 | |> Pipeline.required "isIntersecting" Decode.bool
30 | |> Pipeline.required "intersectionRatio" Decode.float
31 | |> Pipeline.required "distanceToCenter" Decode.float
32 | |> Pipeline.required "targetID" Decode.string
33 |
--------------------------------------------------------------------------------
/frontend/src/Route.elm:
--------------------------------------------------------------------------------
1 | module Route exposing
2 | ( Language
3 | , PagePath
4 | , ProjectName
5 | , Route(..)
6 | , SearchQuery
7 | , SectionID
8 | , toRoute
9 | , toString
10 | )
11 |
12 | import Url
13 | import Url.Builder exposing (QueryParameter)
14 | import Url.Parser exposing ((), (), Parser, custom, map, oneOf, parse, s, string, top)
15 | import Url.Parser.Query as Query
16 |
17 |
18 | type alias SearchQuery =
19 | String
20 |
21 |
22 | type alias ProjectName =
23 | String
24 |
25 |
26 | type alias Language =
27 | String
28 |
29 |
30 | type alias PagePath =
31 | String
32 |
33 |
34 | type alias SectionID =
35 | String
36 |
37 |
38 | type Route
39 | = Home (Maybe SearchQuery)
40 | | Project ProjectName (Maybe SearchQuery)
41 | | ProjectLanguage ProjectName Language (Maybe SearchQuery)
42 | | ProjectLanguagePage ProjectName Language PagePath (Maybe SectionID) (Maybe SearchQuery)
43 | | NotFound
44 |
45 |
46 | toRoute : String -> Route
47 | toRoute string =
48 | case Url.fromString string of
49 | Nothing ->
50 | NotFound
51 |
52 | Just url ->
53 | Maybe.withDefault NotFound (parse routeParser url)
54 |
55 |
56 | toString : Route -> String
57 | toString route =
58 | case route of
59 | Home searchQuery ->
60 | Url.Builder.absolute [] (maybeParam "q" searchQuery)
61 |
62 | Project projectName searchQuery ->
63 | Url.Builder.absolute [ projectName ] (maybeParam "q" searchQuery)
64 |
65 | ProjectLanguage projectName language searchQuery ->
66 | Url.Builder.absolute [ projectName, "-", language ] (maybeParam "q" searchQuery)
67 |
68 | ProjectLanguagePage projectName language pagePath sectionID searchQuery ->
69 | Url.Builder.absolute [ projectName, "-", language, "-", pagePath ]
70 | (List.concat
71 | [ maybeParam "id" sectionID
72 | , maybeParam "q" searchQuery
73 | ]
74 | )
75 |
76 | NotFound ->
77 | Url.Builder.absolute [] []
78 |
79 |
80 | maybeParam : String -> Maybe String -> List QueryParameter
81 | maybeParam name value =
82 | case value of
83 | Just v ->
84 | [ Url.Builder.string name v ]
85 |
86 | Nothing ->
87 | []
88 |
89 |
90 | routeParser : Parser (Route -> a) a
91 | routeParser =
92 | oneOf
93 | [ map Home (top Query.string "q")
94 | , map Project (projectNameParser Query.string "q")
95 | , map ProjectLanguage (projectNameParser s "-" string Query.string "q")
96 | , map
97 | (\projectName language sectionID searchQuery ->
98 | ProjectLanguagePage projectName
99 | language
100 | "/"
101 | sectionID
102 | searchQuery
103 | )
104 | (projectNameParser
105 | s "-"
106 | string
107 | s "-"
108 | Query.string "id"
109 | Query.string "q"
110 | )
111 | , map ProjectLanguagePage
112 | (projectNameParser
113 | s "-"
114 | string
115 | s "-"
116 | pagePathParser
117 | Query.string "id"
118 | Query.string "q"
119 | )
120 | ]
121 |
122 |
123 | projectNameParser : Parser (String -> a) a
124 | projectNameParser =
125 | oneOf
126 | [ map (\a b c d e -> String.join "/" [ a, b, c, d, e ]) (notDash notDash notDash notDash notDash)
127 | , map (\a b c d -> String.join "/" [ a, b, c, d ]) (notDash notDash notDash notDash)
128 | , map (\a b c -> String.join "/" [ a, b, c ]) (notDash notDash notDash)
129 | , map (\a b -> String.join "/" [ a, b ]) (notDash notDash)
130 | , notDash
131 | ]
132 |
133 |
134 | pagePathParser : Parser (String -> a) a
135 | pagePathParser =
136 | oneOf
137 | [ map (\a b c d e f g h i -> String.join "/" [ a, b, c, d, e, f, g, h, i ]) (string string string string string string string string string)
138 | , map (\a b c d e f g h -> String.join "/" [ a, b, c, d, e, f, g, h ]) (string string string string string string string string)
139 | , map (\a b c d e f g -> String.join "/" [ a, b, c, d, e, f, g ]) (string string string string string string string)
140 | , map (\a b c d e f -> String.join "/" [ a, b, c, d, e, f ]) (string string string string string string)
141 | , map (\a b c d e -> String.join "/" [ a, b, c, d, e ]) (string string string string string)
142 | , map (\a b c d -> String.join "/" [ a, b, c, d ]) (string string string string)
143 | , map (\a b c -> String.join "/" [ a, b, c ]) (string string string)
144 | , map (\a b -> String.join "/" [ a, b ]) (string string)
145 | , notDash
146 | ]
147 |
148 |
149 | notDash : Parser (String -> a) a
150 | notDash =
151 | custom "NOT_DASH" <|
152 | \segment ->
153 | if segment /= "-" then
154 | Just segment
155 |
156 | else
157 | Nothing
158 |
--------------------------------------------------------------------------------
/frontend/src/Schema.elm:
--------------------------------------------------------------------------------
1 | module Schema exposing (..)
2 |
3 | import Json.Decode as Decode exposing (Decoder)
4 | import Json.Decode.Pipeline as Pipeline
5 |
6 |
7 | indexDecoder : Decoder Index
8 | indexDecoder =
9 | Decode.succeed Index
10 | |> Pipeline.required "schemaVersion" Decode.string
11 | |> Pipeline.required "directory" Decode.string
12 | |> Pipeline.required "gitRepository" Decode.string
13 | |> Pipeline.required "gitCommitID" Decode.string
14 | |> Pipeline.required "gitRefName" Decode.string
15 | |> Pipeline.required "createdAt" Decode.string
16 | |> Pipeline.required "numFiles" Decode.int
17 | |> Pipeline.required "numBytes" Decode.int
18 | |> Pipeline.required "durationSeconds" Decode.float
19 | |> Pipeline.required "language" languageDecoder
20 | |> Pipeline.required "libraries" (Decode.list libraryDecoder)
21 |
22 |
23 | type alias Index =
24 | { -- The version of the doctree schema in use. Set this to the LatestVersion constant.
25 | schemaVersion : String
26 | , -- Directory that was indexed (absolute path.)
27 | directory : String
28 | , -- GitRepository is the normalized Git repository URI. e.g. "https://github.com/golang/go" or
29 | -- "git@github.com:golang/go" - the same value reported by `git config --get remote.origin.url`
30 | -- with `git@github.com:foo/bar` rewritten to `git://github.com/foo/bar`, credentials removed,
31 | -- any ".git" suffix removed, and any leading "/" prefix removed.
32 | --
33 | -- Empty string if the indexed directory was not a Git repository.
34 | gitRepository : String
35 | , -- GitCommitID is the SHA commit hash of the Git repository revision at the time of indexing, as
36 | -- reported by `git rev-parse HEAD`.
37 | --
38 | -- Empty string if the indexed directory was not a Git repository.
39 | gitCommitID : String
40 | , -- GitRefName is the current Git ref name (branch name, tag name, etc.) as reported by `git rev-parse --abbrev-ref HEAD`
41 | --
42 | -- Empty string if the indexed directory was not a Git repository.
43 | gitRefName : String
44 | , -- CreatedAt time of the index (RFC3339)
45 | createdAt : String
46 | , -- NumFiles indexed.
47 | numFiles : Int
48 | , -- NumBytes indexed.
49 | numBytes : Int
50 | , -- DurationSeconds is how long indexing took.
51 | durationSeconds : Float
52 | , -- Language name
53 | language : Language
54 | , -- Library documentation
55 | libraries : List Library
56 | }
57 |
58 |
59 | languageDecoder : Decoder Language
60 | languageDecoder =
61 | Decode.succeed Language
62 | |> Pipeline.required "title" Decode.string
63 | |> Pipeline.required "id" Decode.string
64 |
65 |
66 | type alias Language =
67 | { -- Title of the language, e.g. "C++" or "Objective-C"
68 | title : String
69 | , -- ID of the language, e.g. "cpp", "objc". Lowercase.
70 | id : String
71 | }
72 |
73 |
74 | libraryDecoder : Decoder Library
75 | libraryDecoder =
76 | Decode.succeed Library
77 | |> Pipeline.required "name" Decode.string
78 | |> Pipeline.required "id" Decode.string
79 | |> Pipeline.required "version" Decode.string
80 | |> Pipeline.required "versionType" Decode.string
81 | |> Pipeline.required "pages" (Decode.list pageDecoder)
82 |
83 |
84 | type alias Library =
85 | { -- Name of the library
86 | name : String
87 | , -- ID of this repository. Many languages have a unique identifier, for example in Java this may
88 | -- be "com.google.android.webview" in Python it may be the PyPi package name. For Rust, the
89 | -- Cargo crate name, etc.
90 | id : String
91 | , -- Version of the library
92 | version : String
93 | , -- Version string type, e.g. "semver", "commit"
94 | versionType : String
95 | , -- Pages of documentation for the library.
96 | pages : List Page
97 | }
98 |
99 |
100 | pageDecoder : Decoder Page
101 | pageDecoder =
102 | Decode.succeed Page
103 | |> Pipeline.required "path" Decode.string
104 | |> Pipeline.required "title" Decode.string
105 | |> Pipeline.required "detail" Decode.string
106 | |> Pipeline.required "searchKey" (Decode.list Decode.string)
107 | |> Pipeline.required "sections" (Decode.lazy (\_ -> sectionsDecoder))
108 | |> Pipeline.optional "subpages" (Decode.lazy (\_ -> pagesDecoder)) (Pages [])
109 |
110 |
111 | type alias Page =
112 | { -- Path of the page relative to the library. This is the URL path and does not necessarily have
113 | -- to match up with filepaths.
114 | path : String
115 | , -- Title of the page.
116 | title : String
117 | , -- The detail
118 | detail : Markdown
119 |
120 | -- SearchKey describes a single string a user would type in to a search bar to find this
121 | -- page. For example, in Go this might be "net/http"
122 | -- This is a list of strings to diffentiate the different "parts" of the string, for Go it would
123 | -- actually be ["net", "/", "http"]. The search engine will do fuzzy prefix/suffix matching of
124 | -- each *part* of the key. For example, a query for "net" would be treated as "*net*".
125 | -- The key should aim to be unique within the scope of the directory and language that was
126 | -- indexed (you can imagine the key is prefixed with the language name and directory/repository
127 | -- name for you.)
128 | , searchKey : List String
129 | , -- Sections of the page.
130 | sections : Sections
131 | , -- Subpages of this one.
132 | subpages : Pages
133 | }
134 |
135 |
136 | type Pages
137 | = Pages (List Page)
138 |
139 |
140 | pagesDecoder =
141 | Decode.map Pages <| Decode.list (Decode.lazy (\_ -> pageDecoder))
142 |
143 |
144 | sectionDecoder : Decoder Section
145 | sectionDecoder =
146 | Decode.succeed Section
147 | |> Pipeline.required "id" Decode.string
148 | |> Pipeline.required "category" Decode.bool
149 | |> Pipeline.required "shortLabel" Decode.string
150 | |> Pipeline.required "label" Decode.string
151 | |> Pipeline.required "detail" Decode.string
152 | |> Pipeline.required "searchKey" (Decode.list Decode.string)
153 | |> Pipeline.optional "children" (Decode.lazy (\_ -> sectionsDecoder)) (Sections [])
154 |
155 |
156 | type alias Section =
157 | { -- The ID of this section, used in the hyperlink to link to this section of the page.
158 | id : String
159 | , -- Category indicates if this section is just describing a category of children, for example
160 | -- if this section has the label "Functions" and Children are all of the functions in the
161 | -- library. This information is used to pick out key sections that should be shown in high-level
162 | -- navigation.
163 | category : Bool
164 | , -- ShortLabel is the shortest string that can describe this section relative to the parent. For
165 | -- example, in Go this may be `(r) GetName` as a reduced form of `func (r *Route) GetName`.
166 | shortLabel : String
167 | , -- The label of this section.
168 | label : Markdown
169 | , -- The detail
170 | detail : Markdown
171 |
172 | -- SearchKey describes a single string a user would type in to a search bar to find this
173 | -- section. For example, in Go this might be "net/http.Client.PostForm"
174 | --
175 | -- This is a list of strings to diffentiate the different "parts" of the string, for Go it would
176 | -- actually be ["net", "/", "http", ".", "Client", ".", "PostForm"]. The search engine will do
177 | -- fuzzy prefix/suffix matching of each *part* of the key. For example, a query for
178 | -- "net.PostForm" would be treated as "*net*.*PostForm*".
179 | --
180 | -- The key should aim to be unique within the scope of the directory and language that was
181 | -- indexed (you can imagine the key is prefixed with the language name and directory/repository
182 | -- name for you.)
183 | , searchKey : List String
184 | , -- Any children sections. For example, if this section represents a class the children could be
185 | -- the methods of the class and they would be rendered immediately below this section and
186 | -- indicated as being children of the parent section.
187 | children : Sections
188 | }
189 |
190 |
191 | type Sections
192 | = Sections (List Section)
193 |
194 |
195 | sectionsDecoder =
196 | Decode.map Sections <| Decode.list (Decode.lazy (\_ -> sectionDecoder))
197 |
198 |
199 | type alias Markdown =
200 | String
201 |
--------------------------------------------------------------------------------
/frontend/src/Search.elm:
--------------------------------------------------------------------------------
1 | module Search exposing (..)
2 |
3 | import API
4 | import APISchema
5 | import Browser.Dom
6 | import Element as E
7 | import Element.Border as Border
8 | import Element.Font as Font
9 | import Html
10 | import Html.Attributes
11 | import Html.Events
12 | import Http
13 | import Process
14 | import Task
15 | import Url.Builder
16 | import Util exposing (httpErrorToString)
17 |
18 |
19 | debounceQueryInputMillis : Float
20 | debounceQueryInputMillis =
21 | 20
22 |
23 |
24 | debounceQueryIntentInputMillis : Float
25 | debounceQueryIntentInputMillis =
26 | 500
27 |
28 |
29 |
30 | -- INIT
31 |
32 |
33 | type alias Model =
34 | { debounce : Int
35 | , debounceIntent : Int
36 | , query : String
37 | , projectName : Maybe String
38 | , results : Maybe (Result Http.Error APISchema.SearchResults)
39 | }
40 |
41 |
42 | init : Maybe String -> ( Model, Cmd Msg )
43 | init projectName =
44 | ( { debounce = 0
45 | , debounceIntent = 0
46 | , query = ""
47 | , projectName = projectName
48 | , results = Nothing
49 | }
50 | , Task.perform
51 | (\_ -> FocusOn "search-input")
52 | (Process.sleep 100)
53 | )
54 |
55 |
56 |
57 | -- UPDATE
58 |
59 |
60 | type Msg
61 | = FocusOn String
62 | | OnSearchInput String
63 | | OnDebounce
64 | | OnDebounceIntent
65 | | RunSearch Bool
66 | | GotSearchResults (Result Http.Error APISchema.SearchResults)
67 | | NoOp
68 |
69 |
70 | update : Msg -> Model -> ( Model, Cmd Msg )
71 | update msg model =
72 | case msg of
73 | OnSearchInput query ->
74 | ( { model
75 | | query = query
76 | , debounce = model.debounce + 1
77 | , debounceIntent = model.debounceIntent + 1
78 | }
79 | , Cmd.batch
80 | [ Task.perform (\_ -> OnDebounce) (Process.sleep debounceQueryInputMillis)
81 | , Task.perform (\_ -> OnDebounceIntent) (Process.sleep debounceQueryIntentInputMillis)
82 | ]
83 | )
84 |
85 | OnDebounce ->
86 | if model.debounce - 1 == 0 then
87 | update (RunSearch False) { model | debounce = model.debounce - 1 }
88 |
89 | else
90 | ( { model | debounce = model.debounce - 1 }, Cmd.none )
91 |
92 | OnDebounceIntent ->
93 | if model.debounceIntent - 1 == 0 then
94 | update (RunSearch True) { model | debounceIntent = model.debounceIntent - 1 }
95 |
96 | else
97 | ( { model | debounceIntent = model.debounceIntent - 1 }, Cmd.none )
98 |
99 | RunSearch intent ->
100 | ( model, API.fetchSearchResults GotSearchResults model.query intent model.projectName )
101 |
102 | GotSearchResults results ->
103 | ( { model | results = Just results }, Cmd.none )
104 |
105 | FocusOn id ->
106 | ( model, Browser.Dom.focus id |> Task.attempt (\_ -> NoOp) )
107 |
108 | NoOp ->
109 | ( model, Cmd.none )
110 |
111 |
112 |
113 | -- VIEW
114 |
115 |
116 | searchInput =
117 | E.html
118 | (Html.input
119 | [ Html.Attributes.type_ "text"
120 | , Html.Attributes.autofocus True
121 | , Html.Attributes.id "search-input"
122 | , Html.Attributes.placeholder "go http.ListenAndServe"
123 | , Html.Attributes.style "font-size" "16px"
124 | , Html.Attributes.style "font-family" "JetBrains Mono, monospace"
125 | , Html.Attributes.style "padding" "0.5rem"
126 | , Html.Attributes.style "width" "100%"
127 | , Html.Attributes.style "margin-bottom" "2rem"
128 | , Html.Events.onInput OnSearchInput
129 | ]
130 | []
131 | )
132 |
133 |
134 | searchResults : Maybe (Result Http.Error APISchema.SearchResults) -> E.Element msg
135 | searchResults request =
136 | case request of
137 | Just response ->
138 | case response of
139 | Ok results ->
140 | E.column [ E.width E.fill ]
141 | (List.map
142 | (\r ->
143 | E.row
144 | [ E.width E.fill
145 | , E.paddingXY 0 8
146 | , Border.color (E.rgb255 210 210 210)
147 | , Border.widthEach { top = 0, left = 0, bottom = 1, right = 0 }
148 | ]
149 | [ E.column []
150 | [ E.link [ E.paddingEach { top = 0, right = 0, bottom = 4, left = 0 } ]
151 | { url = Url.Builder.absolute [ r.projectName, "-", r.language, "-", r.path ] [ Url.Builder.string "id" r.id ]
152 | , label = E.el [ Font.underline ] (E.text r.searchKey)
153 | }
154 | , E.el
155 | [ Font.color (E.rgb 0.6 0.6 0.6)
156 | , Font.size 14
157 | ]
158 | (E.text (Util.shortProjectName r.path))
159 | ]
160 | , E.el
161 | [ E.alignRight
162 | , Font.color (E.rgb 0.6 0.6 0.6)
163 | , Font.size 14
164 | ]
165 | (E.text (Util.shortProjectName r.projectName))
166 | ]
167 | )
168 | results
169 | )
170 |
171 | Err err ->
172 | E.text (httpErrorToString err)
173 |
174 | Nothing ->
175 | E.text "loading.."
176 |
--------------------------------------------------------------------------------
/frontend/src/Style.elm:
--------------------------------------------------------------------------------
1 | module Style exposing (..)
2 |
3 | import Element as E
4 | import Element.Border as Border
5 | import Element.Font as Font
6 | import Element.Region as Region
7 |
8 |
9 | font =
10 | Font.family [ Font.typeface "JetBrains Mono", Font.monospace ]
11 |
12 |
13 | fontSize =
14 | Font.size 16
15 |
16 |
17 | layout =
18 | [ font, fontSize ]
19 |
20 |
21 | h1 attrs child =
22 | E.paragraph (List.concat [ attrs, [ Region.heading 1, Font.size 32, Font.bold ] ]) [ child ]
23 |
24 |
25 | h2 attrs child =
26 | E.paragraph (List.concat [ attrs, [ Region.heading 2, Font.size 24, Font.bold ] ]) [ child ]
27 |
28 |
29 | h3 attrs child =
30 | E.paragraph (List.concat [ attrs, [ Region.heading 3, Font.size 20, Font.bold ] ]) [ child ]
31 |
32 |
33 | h4 attrs child =
34 | E.paragraph (List.concat [ attrs, [ Region.heading 4, Font.size 16, Font.bold ] ]) [ child ]
35 |
36 |
37 | paragraph attrs child =
38 | E.paragraph (List.concat [ attrs, [ Font.size 16, Font.family [ Font.typeface "Verdana, Geneva, sans-serif" ] ] ]) child
39 |
40 |
41 | tableHeader child =
42 | E.el
43 | [ E.paddingXY 8 8
44 | , Font.bold
45 | , Border.color (E.rgb255 210 210 210)
46 | , Border.widthEach { top = 0, left = 1, bottom = 0, right = 0 }
47 | ]
48 | child
49 |
50 |
51 | tableCell child =
52 | E.el
53 | [ E.paddingXY 8 8
54 | , Border.color (E.rgb255 210 210 210)
55 | , Border.widthEach { top = 1, left = 1, bottom = 0, right = 0 }
56 | ]
57 | child
58 |
--------------------------------------------------------------------------------
/frontend/src/Util.elm:
--------------------------------------------------------------------------------
1 | module Util exposing (..)
2 |
3 | import Http
4 |
5 |
6 | httpErrorToString : Http.Error -> String
7 | httpErrorToString error =
8 | case error of
9 | Http.BadUrl url ->
10 | "The URL " ++ url ++ " was invalid"
11 |
12 | Http.Timeout ->
13 | "Unable to reach the server, try again"
14 |
15 | Http.NetworkError ->
16 | "Unable to reach the server, check your network connection"
17 |
18 | Http.BadStatus 500 ->
19 | "The server had a problem, try again later"
20 |
21 | Http.BadStatus 400 ->
22 | "Verify your information and try again"
23 |
24 | Http.BadStatus _ ->
25 | "Unknown error"
26 |
27 | Http.BadBody errorMessage ->
28 | errorMessage
29 |
30 |
31 | maybeEquals : a -> a -> Maybe a
32 | maybeEquals v1 v2 =
33 | if v1 == v2 then
34 | Just v1
35 |
36 | else
37 | Nothing
38 |
39 |
40 | shortProjectName : String -> String
41 | shortProjectName name =
42 | trimPrefix name "github.com/"
43 |
44 |
45 | trimPrefix : String -> String -> String
46 | trimPrefix str prefix =
47 | if String.startsWith prefix str then
48 | String.dropLeft (String.length prefix) str
49 |
50 | else
51 | str
52 |
53 |
54 | trimSuffix : String -> String -> String
55 | trimSuffix str suffix =
56 | if String.endsWith suffix str then
57 | String.dropRight (String.length suffix) str
58 |
59 | else
60 | str
61 |
62 |
63 | boolToString : Bool -> String
64 | boolToString value =
65 | if value then
66 | "true"
67 |
68 | else
69 | "false"
70 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/sourcegraph/doctree
2 |
3 | go 1.18
4 |
5 | require (
6 | github.com/DaivikDave/tree-sitter-jsdoc/bindings/go v0.0.0-20220602053452-e02513edad7b
7 | github.com/NYTimes/gziphandler v1.1.1
8 | github.com/adrg/frontmatter v0.2.0
9 | github.com/agnivade/levenshtein v1.1.1
10 | github.com/fsnotify/fsnotify v1.5.4
11 | github.com/hashicorp/go-multierror v1.1.1
12 | github.com/hexops/autogold v1.3.0
13 | github.com/hexops/cmder v1.0.1
14 | github.com/pkg/errors v0.9.1
15 | github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3
16 | github.com/slimsag/tree-sitter-zig/bindings/go v0.0.0-20220513090138-e3dbdff9d013
17 | github.com/smacker/go-tree-sitter v0.0.0-20220611151427-2c4b54ed41fe
18 | github.com/spaolacci/murmur3 v1.1.0
19 | )
20 |
21 | require (
22 | github.com/BurntSushi/toml v1.1.0 // indirect
23 | github.com/davecgh/go-spew v1.1.1 // indirect
24 | github.com/google/go-cmp v0.5.7 // indirect
25 | github.com/hashicorp/errwrap v1.1.0 // indirect
26 | github.com/hexops/gotextdiff v1.0.3 // indirect
27 | github.com/hexops/valast v1.4.0 // indirect
28 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
29 | github.com/nightlyone/lockfile v1.0.0 // indirect
30 | github.com/shurcooL/go-goon v0.0.0-20210110234559-7585751d9a17 // indirect
31 | golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
32 | golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect
33 | golang.org/x/tools v0.1.10 // indirect
34 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
35 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
36 | gopkg.in/yaml.v2 v2.4.0 // indirect
37 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
38 | mvdan.cc/gofumpt v0.3.0 // indirect
39 | )
40 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
2 | github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
3 | github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
4 | github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I=
5 | github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
6 | github.com/DaivikDave/tree-sitter-jsdoc/bindings/go v0.0.0-20220602053452-e02513edad7b h1:fprQbldp/8U06ehCEQjZZ6B21RvRmHCpcNYbemuQpGM=
7 | github.com/DaivikDave/tree-sitter-jsdoc/bindings/go v0.0.0-20220602053452-e02513edad7b/go.mod h1:bgtG4OF+KG6FD9luXtrFyoKPdOqBM8JEIVlLFod8Cgo=
8 | github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
9 | github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
10 | github.com/adrg/frontmatter v0.2.0 h1:/DgnNe82o03riBd1S+ZDjd43wAmC6W35q67NHeLkPd4=
11 | github.com/adrg/frontmatter v0.2.0/go.mod h1:93rQCj3z3ZlwyxxpQioRKC1wDLto4aXHrbqIsnH9wmE=
12 | github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
13 | github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
14 | github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
15 | github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
16 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
17 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
18 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
19 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
20 | github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g=
21 | github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
22 | github.com/frankban/quicktest v1.14.2 h1:SPb1KFFmM+ybpEjPUhCCkZOM5xlovT5UbrMvWnXyBns=
23 | github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
24 | github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
25 | github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
26 | github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
27 | github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
28 | github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
29 | github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
30 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
31 | github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
32 | github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
33 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
34 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
35 | github.com/hexops/autogold v0.8.1/go.mod h1:97HLDXyG23akzAoRYJh/2OBs3kd80eHyKPvZw0S5ZBY=
36 | github.com/hexops/autogold v1.3.0 h1:IEtGNPxBeBu8RMn8eKWh/Ll9dVNgSnJ7bp/qHgMQ14o=
37 | github.com/hexops/autogold v1.3.0/go.mod h1:d4hwi2rid66Sag+BVuHgwakW/EmaFr8vdTSbWDbrDRI=
38 | github.com/hexops/cmder v1.0.1 h1:zCrJUwlXvYHGE0tMAXMxkhRRcL0NMKlCxaiXm+IoheY=
39 | github.com/hexops/cmder v1.0.1/go.mod h1:7vsl5I9EK1NGtWYgBzQtOM4e13WReMsXpogtGhZWzAg=
40 | github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
41 | github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
42 | github.com/hexops/valast v1.4.0 h1:sFzyxPDP0riFQUzSBXTCCrAbbIndHPWMndxuEjXdZlc=
43 | github.com/hexops/valast v1.4.0/go.mod h1:uVjKZ0smVuYlgCSPz9NRi5A04sl7lp6GtFWsROKDgEs=
44 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
45 | github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
46 | github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
47 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
48 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
49 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
50 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
51 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
52 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
53 | github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA=
54 | github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI=
55 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
56 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
57 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
58 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
59 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
60 | github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
61 | github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
62 | github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg=
63 | github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
64 | github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
65 | github.com/shurcooL/go-goon v0.0.0-20210110234559-7585751d9a17 h1:lRAUE0dIvigSSFAmaM2dfg7OH8T+a8zJ5smEh09a/GI=
66 | github.com/shurcooL/go-goon v0.0.0-20210110234559-7585751d9a17/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
67 | github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3 h1:sAARUcYbwxnebBeWHzKX2MeyXtzy25TEglCTz9BhueY=
68 | github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3/go.mod h1:AIBPxLCkKUFc2ZkjCXzs/Kk9OUhQLw/Zicdd0Rhqz2U=
69 | github.com/slimsag/tree-sitter-zig/bindings/go v0.0.0-20220513090138-e3dbdff9d013 h1:ReRFigxtkfnm9L3A2y7+eBaL9OqBwfRRNcq451gNQJY=
70 | github.com/slimsag/tree-sitter-zig/bindings/go v0.0.0-20220513090138-e3dbdff9d013/go.mod h1:GDd3OCJ/fgx5gLuEIrfOMJ7SGPlQP3ZTYdyaUcvpNyg=
71 | github.com/smacker/go-tree-sitter v0.0.0-20220421092837-ec55f7cfeaf4 h1:UFOHRX5nrxNCVORhicjy31nzSVt9rEjf/YRcx2Dc3MM=
72 | github.com/smacker/go-tree-sitter v0.0.0-20220421092837-ec55f7cfeaf4/go.mod h1:EiUuVMUfLQj8Sul+S8aKWJwQy7FRYnJCO2EWzf8F5hk=
73 | github.com/smacker/go-tree-sitter v0.0.0-20220611151427-2c4b54ed41fe h1:5PlhnnZ/kBl2Y3TTh1LwT4ivLWWCqZGcfyom5t3DydQ=
74 | github.com/smacker/go-tree-sitter v0.0.0-20220611151427-2c4b54ed41fe/go.mod h1:EiUuVMUfLQj8Sul+S8aKWJwQy7FRYnJCO2EWzf8F5hk=
75 | github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
76 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
77 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
78 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
79 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
80 | github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
81 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
82 | github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
83 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
84 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
85 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
86 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
87 | golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
88 | golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
89 | golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
90 | golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
91 | golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
92 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
93 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
94 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
95 | golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
96 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
97 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
98 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
99 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
100 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
101 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
102 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
103 | golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
104 | golang.org/x/sys v0.0.0-20210218084038-e8e29180ff58/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
105 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
106 | golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
107 | golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
108 | golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
109 | golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
110 | golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c h1:aFV+BgZ4svzjfabn8ERpuB4JI4N6/rdy1iusx77G3oU=
111 | golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
112 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
113 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
114 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
115 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
116 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
117 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
118 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
119 | golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
120 | golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
121 | golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
122 | golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
123 | golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
124 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
125 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
126 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
127 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
128 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
129 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
130 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
131 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
132 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
133 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
134 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
135 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
136 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
137 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
138 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
139 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
140 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
141 | mvdan.cc/gofumpt v0.0.0-20210107193838-d24d34e18d44/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=
142 | mvdan.cc/gofumpt v0.1.0/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=
143 | mvdan.cc/gofumpt v0.3.0 h1:kTojdZo9AcEYbQYhGuLf/zszYthRdhDNDUi2JKTxas4=
144 | mvdan.cc/gofumpt v0.3.0/go.mod h1:0+VyGZWleeIj5oostkOex+nDBA0eyavuDnDusAJ8ylo=
145 |
--------------------------------------------------------------------------------
/renovate.json:
--------------------------------------------------------------------------------
1 | {
2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json",
3 | "extends": [
4 | "local>sourcegraph/renovate-config"
5 | ]
6 | }
7 |
--------------------------------------------------------------------------------