├── .bazelrc ├── .bazelrc.circle ├── .circleci ├── Dockerfile ├── build.sh └── config.yml ├── .gitignore ├── BUILD ├── COPYING ├── README.md ├── WORKSPACE ├── blameworthy ├── BUILD ├── gitops.go ├── gitops_test.go ├── indexer.go ├── indexer_test.go └── test_data │ ├── git-log.dashing │ └── git-log.dashing.stripped ├── client └── test │ ├── BUILD │ ├── bench_test.go │ ├── integration_test.go │ ├── patterns │ ├── suites │ ├── basic │ ├── benchmarks │ ├── index │ ├── slow │ ├── testcases │ └── unicode │ └── testutil.go ├── cmd ├── BUILD ├── lg │ ├── BUILD │ └── main.go ├── livegrep-fetch-reindex │ ├── BUILD │ └── main.go ├── livegrep-git-log │ ├── BUILD │ └── main.go ├── livegrep-github-reindex │ ├── BUILD │ ├── flags.go │ └── main.go ├── livegrep-reload │ ├── BUILD │ └── main.go ├── livegrep-update-blame-cache │ ├── BUILD │ └── main.go └── livegrep │ ├── BUILD │ └── livegrep.go ├── doc └── examples │ └── livegrep │ ├── generate_ordered_contents.sh │ ├── index.json │ ├── index_with_ordered_contents.json │ ├── ordered-contents.txt │ └── server.json ├── jsonframe ├── BUILD └── jsonframe.go ├── package.sh ├── server ├── BUILD ├── api.go ├── api │ ├── BUILD │ └── types.go ├── backend.go ├── config │ ├── BUILD │ └── config.go ├── fastforward.go ├── fastforward_test.go ├── fileblame.go ├── fileview.go ├── json.go ├── log │ ├── BUILD │ └── log.go ├── middleware │ ├── BUILD │ └── reverse_proxy.go ├── query.go ├── query_test.go ├── reqid │ ├── BUILD │ └── reqid.go ├── server.go ├── server_test.go └── templates │ ├── BUILD │ └── templates.go ├── src ├── BUILD ├── chunk.cc ├── chunk.h ├── chunk_allocator.cc ├── chunk_allocator.h ├── codesearch.cc ├── codesearch.h ├── common.h ├── content.cc ├── content.h ├── dump_load.cc ├── dump_load.h ├── fs_indexer.cc ├── fs_indexer.h ├── git_indexer.cc ├── git_indexer.h ├── lib │ ├── BUILD │ ├── debug.cc │ ├── debug.h │ ├── metrics.cc │ ├── metrics.h │ ├── per_thread.h │ ├── radix_sort.cc │ ├── radix_sort.h │ ├── recursion.h │ ├── thread_queue.h │ └── timer.h ├── proto │ ├── BUILD │ ├── config.proto │ └── livegrep.proto ├── query_planner.cc ├── query_planner.h ├── re_width.cc ├── re_width.h ├── smart_git.h ├── tagsearch.cc ├── tagsearch.h └── tools │ ├── BUILD │ ├── analyze-re.cc │ ├── codesearch.cc │ ├── codesearchtool.cc │ ├── dump-file.cc │ ├── grpc_server.cc │ ├── grpc_server.h │ ├── inspect-index.cc │ └── limits.h ├── test ├── .gitignore ├── BUILD ├── codesearch_test.cc ├── main.cc ├── planner_test.cc └── tagsearch_test.cc ├── third_party ├── BUILD ├── BUILD.divsufsort ├── BUILD.gtest ├── BUILD.libgit2 ├── divsufsort.patch ├── sparseconfig.h └── utf8cpp │ ├── doc │ ├── ReleaseNotes │ └── utf8cpp.html │ └── source │ ├── utf8.h │ └── utf8 │ ├── checked.h │ ├── core.h │ └── unchecked.h ├── tools └── build_defs │ ├── BUILD │ ├── go_externals.bzl │ └── libgit2.bzl ├── travisdeps.sh └── web ├── 3d ├── BUILD └── html.js ├── BUILD ├── htdocs └── assets │ ├── 3d │ ├── README.bootstrap │ └── bootstrap.min.css │ ├── css │ ├── blame.css │ └── codesearch.css │ ├── img │ └── favicon.ico │ └── raw-js │ ├── analytics.js │ └── blame.js ├── npm ├── css-loader │ ├── BUILD │ └── npm-shrinkwrap.json └── style-loader │ ├── BUILD │ └── npm-shrinkwrap.json ├── src ├── codesearch │ ├── codesearch.js │ ├── codesearch_ui.js │ └── repo_selector.js ├── entry.js ├── fileview │ └── fileview.js └── prism.js ├── templates ├── about.html ├── blamediff.html ├── blamefile.html ├── blamemessage.html ├── common │ └── layout.html ├── fileview.html ├── index.html ├── logfile.html └── opensearch.xml └── webpack.config.js /.bazelrc: -------------------------------------------------------------------------------- 1 | # c.f. https://github.com/grpc/grpc/pull/13929 2 | build --copt=-DGRPC_BAZEL_BUILD 3 | -------------------------------------------------------------------------------- /.bazelrc.circle: -------------------------------------------------------------------------------- 1 | # This is from Bazel's former travis setup, to avoid blowing up the RAM usage. 2 | startup --host_jvm_args=-Xms512m 3 | startup --host_jvm_args=-Xmx1024m 4 | build --local_resources=4096,4,1.0 5 | build -c opt 6 | test --ram_utilization_factor=10 7 | 8 | # This is so we understand failures better 9 | build --verbose_failures 10 | -------------------------------------------------------------------------------- /.circleci/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 2 | 3 | RUN apt-get update && apt-get -y install \ 4 | build-essential \ 5 | libxml2-utils \ 6 | wget \ 7 | pkg-config \ 8 | zip \ 9 | unzip \ 10 | zlib1g-dev \ 11 | openjdk-8-jdk \ 12 | git \ 13 | openssh-client \ 14 | python 15 | 16 | ADD https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-173.0.0-linux-x86_64.tar.gz /tmp/gcloud.tar.gz 17 | RUN tar -C /usr/local/ -xzf /tmp/gcloud.tar.gz 18 | RUN /usr/local/google-cloud-sdk/install.sh 19 | 20 | RUN wget --quiet -O /tmp/bazel-0.21.0-installer-linux-x86_64.sh \ 21 | 'https://github.com/bazelbuild/bazel/releases/download/0.21.0/bazel-0.21.0-installer-linux-x86_64.sh' && \ 22 | chmod +x /tmp/bazel-0.21.0-installer-linux-x86_64.sh 23 | RUN /tmp/bazel-0.21.0-installer-linux-x86_64.sh 24 | -------------------------------------------------------------------------------- /.circleci/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | if [ "$GCLOUD_SERVICE_KEY" ]; then 5 | echo "$GCLOUD_SERVICE_KEY" | base64 --decode --ignore-garbage > ${HOME}/gcloud-service-key.json 6 | /usr/local/google-cloud-sdk/bin/gcloud auth activate-service-account --key-file ${HOME}/gcloud-service-key.json 7 | /usr/local/google-cloud-sdk/bin/gcloud config set project livegrep 8 | fi 9 | 10 | cat .bazelrc.circle >> .bazelrc 11 | 12 | bazel fetch //cmd/... 13 | 14 | gofmt=$(bazel info output_base)/external/go_sdk/bin/gofmt 15 | format_errors=$(find . -name '*.go' -print0 | xargs -0 "$gofmt" -l -e) 16 | if [ "$format_errors" ]; then 17 | echo "=== misformatted files (run gofmt) ===" 18 | echo "$format_errors" 19 | exit 1 20 | fi 21 | 22 | bazel test --test_arg=-test.v //... 23 | bazel build //... 24 | 25 | # bazel-bin/client/test/go_default_test -test.repo "$(pwd)/deps/linux" 26 | 27 | if [ "$GCLOUD_SERVICE_KEY" ]; then 28 | ./package.sh 29 | /usr/local/google-cloud-sdk/bin/gsutil cp -a public-read -r builds/ gs://livegrep.appspot.com/ 30 | fi 31 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | docker: 5 | - image: us.gcr.io/livegrep/ci:7 6 | 7 | steps: 8 | - checkout 9 | - restore_cache: 10 | keys: 11 | - bazel-cache 12 | 13 | - run: ./.circleci/build.sh 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.a 2 | *.o 3 | *.test 4 | .*.d 5 | /.gopath 6 | /.makevars/ 7 | /.vagrant 8 | /Makefile.config 9 | /bin/ 10 | /node_modules 11 | /repos/ 12 | /src/vendor/libdivsufsort/build/ 13 | /web/log/ 14 | config.json 15 | vendor/re2/obj/libre2.a 16 | /bazel-* 17 | compile_commands.json 18 | .devbox 19 | -------------------------------------------------------------------------------- /BUILD: -------------------------------------------------------------------------------- 1 | load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") 2 | load("@compdb//:aspects.bzl", "compilation_database") 3 | 4 | compilation_database( 5 | name = "compilation_db", 6 | targets = [ 7 | "//src/tools:codesearch", 8 | "//src/tools:codesearchtool", 9 | ], 10 | ) 11 | 12 | load("@bazel_gazelle//:def.bzl", "gazelle") 13 | 14 | # gazelle:prefix github.com/livegrep/livegrep 15 | gazelle(name = "gazelle") 16 | 17 | filegroup( 18 | name = "docs", 19 | srcs = glob([ 20 | "doc/**/*", 21 | ]), 22 | ) 23 | 24 | pkg_tar( 25 | name = "livegrep", 26 | srcs = [ 27 | ":COPYING", 28 | ":README.md", 29 | ":docs", 30 | ], 31 | strip_prefix = ".", 32 | deps = [ 33 | "//cmd:go_tools", 34 | "//src/tools:cc_tools", 35 | "//web:assets", 36 | ], 37 | ) 38 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011-2013 Nelson Elhage 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | The views and conclusions contained in the software and documentation are those 25 | of the authors and should not be interpreted as representing official policies, 26 | either expressed or implied, of the author. 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Livegrep [![Build Status](https://circleci.com/gh/livegrep/livegrep.png?branch=master)](https://circleci.com/gh/livegrep/livegrep) 2 | ======== 3 | 4 | Livegrep is a tool, partially inspired by Google Code Search, for 5 | interactive regex search of ~gigabyte-scale source repositories. You 6 | can see a running instance at 7 | [http://livegrep.com/](http://livegrep.com). 8 | 9 | Building 10 | -------- 11 | 12 | livegrep builds using [bazel][bazel]. You will need to 13 | [install bazel][bazel-install] version 0.10 or newer. livegrep vendors 14 | and/or fetches all of its dependencies using `bazel`, and so should 15 | only require a relatively recent C++ compiler to build. 16 | 17 | Once you have those dependencies, you can build using 18 | 19 | bazel build //... 20 | 21 | Note that the initial build will download around 100M of 22 | dependencies. These will be cached once downloaded. 23 | 24 | [bazel]: http://www.bazel.io/ 25 | [bazel-install]: http://www.bazel.io/docs/install.html 26 | 27 | Invoking 28 | -------- 29 | 30 | To run `livegrep`, you need to invoke both the `codesearch` backend 31 | index/search process, and the `livegrep` web interface. 32 | 33 | To run the sample web interface over livegrep itself, once you have 34 | built both `codesearch` and `livegrep`: 35 | 36 | In one terminal, start the `codesearch` server like so: 37 | 38 | bazel-bin/src/tools/codesearch -grpc localhost:9999 doc/examples/livegrep/index.json 39 | 40 | In another, run livegrep: 41 | 42 | bazel-bin/cmd/livegrep/livegrep 43 | 44 | In a browser, now visit 45 | [http://localhost:8910/](http://localhost:8910/), and you should see a 46 | working livegrep. 47 | 48 | ## Using Index Files 49 | 50 | The `codesearch` binary is responsible for reading source code, 51 | maintaining an index, and handling searches. `livegrep` is stateless 52 | and relies only on the connection to `codesearch` over a TCP 53 | connection. 54 | 55 | By default, `codesearch` will build an in-memory index over the 56 | repositories specified in its configuration file. You can, however, 57 | also instruct it to save the index to a file on disk. This the dual 58 | advantages of allowing indexes that are too large to fit in RAM, and 59 | of allowing an index file to be reused. You instruct `codesearch` to 60 | generate an index file via the `-dump_index` flag and to not launch 61 | a search server via the `-index_only` flag: 62 | 63 | bazel-bin/src/tools/codesearch -index_only -dump_index livegrep.idx doc/examples/livegrep/index.json 64 | 65 | Once `codeseach` has built the index, this index file can be used for 66 | future runs. Index files are standalone, and you no longer need access 67 | to the source code repositories, or even a configuration file, once an 68 | index has been built. You can just launch a search server like so: 69 | 70 | bazel-bin/src/tools/codesearch -load_index livegrep.idx -grpc localhost:9999 71 | 72 | The schema for the `codesearch` configuration file defined using 73 | protobuf in `src/proto/config.proto`. 74 | 75 | ## `livegrep` 76 | 77 | The `livegrep` frontend expects an optional position argument 78 | indicating a JSON configuration file; See 79 | [doc/examples/livegrep/server.json][server.json] for an example, and 80 | [server/config/config.go][config.go] for documentation of available 81 | options. 82 | 83 | By default, `livegrep` will connect to a single local codesearch 84 | instance on port `9999`, and listen for HTTP connections on port 85 | `8910`. 86 | 87 | [server.json]: https://github.com/livegrep/livegrep/blob/master/doc/examples/livegrep/server.json 88 | [config.go]: https://github.com/livegrep/livegrep/blob/master/server/config/config.go 89 | 90 | ## github integration 91 | 92 | `livegrep` includes a helper driver, `livegrep-github-reindex`, which 93 | can automatically update and index selected github repositories. To 94 | download and index all of my repositories (except for forks), storing 95 | the repos in `repos/` and writing `nelhage.idx`, you might run: 96 | 97 | bazel-bin/cmd/livegrep-github-reindex/livegrep-github-reindex -user=nelhage -forks=false -name=github.com/nelhage -out nelhage.idx 98 | 99 | You can now use `nelhage.idx` as an argument to `codesearch 100 | -load_index`. 101 | 102 | Docker images 103 | ------------- 104 | 105 | I build [docker images][docker] for livegrep out of the 106 | [livegrep.com](https://github.com/livegrep/livegrep.com) repository, 107 | based on build images created by this repository's CI. They should be 108 | generally usable. For instance, to build+run a livegrep index of this 109 | repository, you could run: 110 | 111 | ``` 112 | docker run -v $(pwd):/data livegrep/indexer /livegrep/bin/livegrep-github-reindex -repo livegrep/livegrep -http -dir /data 113 | docker network create livegrep 114 | docker run -v $(pwd):/data --network livegrep livegrep/base /livegrep/bin/codesearch -load_index /data/livegrep.idx -grpc 0.0.0.0:9999 115 | docker run -d --network livegrep --publish 8910:8910 livegrep/base /livegrep/bin/livegrep -docroot /livegrep/web -listen=0.0.0.0:8910 116 | ``` 117 | 118 | And then access http://localhost:8910/ 119 | 120 | You can also find the [docker-compose config powering 121 | livegrep.com][docker-compose] in that same repository. 122 | 123 | [docker]: https://hub.docker.com/u/livegrep 124 | [docker-compose]: https://github.com/livegrep/livegrep.com/tree/master/compose 125 | 126 | Resource Usage 127 | -------------- 128 | 129 | livegrep builds an index file of your source code, and then works 130 | entirely out of that index, with no further access to the original git 131 | repositories. 132 | 133 | The index file will vary somewhat in size, but will usually be 3-5x 134 | the size of the indexed text. `livegrep` memory-maps the index file 135 | into RAM, so it can work out of index files larger than (available) 136 | RAM, but will perform better if the file can be loaded entirely into 137 | memory. Barring that, keeping the disk on fast SSDs is recommended for 138 | optimal performance. 139 | 140 | 141 | LICENSE 142 | ------- 143 | 144 | Livegrep is open source. See COPYING for more information. 145 | -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | workspace(name = "com_github_livegrep_livegrep") 2 | 3 | load( 4 | "@bazel_tools//tools/build_defs/repo:git.bzl", 5 | "git_repository", 6 | "new_git_repository", 7 | ) 8 | 9 | load( 10 | "@bazel_tools//tools/build_defs/repo:http.bzl", 11 | "http_archive", 12 | ) 13 | 14 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 15 | 16 | http_archive( 17 | name = "build_stack_rules_proto", 18 | sha256 = "128c4486b1707db917411c6e448849dd76ea3b8ba704f9e0627d9b01f2ee45fe", 19 | strip_prefix = "rules_proto-f5d6eea6a4528bef3c1d3a44d486b51a214d61c2", 20 | urls = ["https://github.com/stackb/rules_proto/archive/f5d6eea6a4528bef3c1d3a44d486b51a214d61c2.tar.gz"], 21 | ) 22 | 23 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 24 | 25 | http_archive( 26 | name = "divsufsort", 27 | build_file = "//third_party:BUILD.divsufsort", 28 | patch_args = ["-p1"], 29 | patches = ["//third_party:divsufsort.patch"], 30 | sha256 = "9164cb6044dcb6e430555721e3318d5a8f38871c2da9fd9256665746a69351e0", 31 | strip_prefix = "libdivsufsort-2.0.1", 32 | type = "tgz", 33 | url = "https://codeload.github.com/y-256/libdivsufsort/tar.gz/2.0.1", 34 | ) 35 | 36 | git_repository( 37 | name = "com_googlesource_code_re2", 38 | commit = "767de83bb7e4bfe3a2d8aec0ec79f9f1f66da30a", 39 | remote = "https://github.com/google/re2", 40 | ) 41 | 42 | git_repository( 43 | name = "gflags", 44 | commit = "e171aa2d15ed9eb17054558e0b3a6a413bb01067", # v2.2.2 45 | remote = "https://github.com/gflags/gflags", 46 | ) 47 | 48 | git_repository( 49 | name = "com_github_nelhage_rules_boost", 50 | commit = "c1d618315fa152958baef8ea0d77043eebf7f573", 51 | remote = "https://github.com/nelhage/rules_boost", 52 | ) 53 | # local_repository( 54 | # name = "com_github_nelhage_boost", 55 | # path = "../rules_boost", 56 | # ) 57 | 58 | load( 59 | "@com_github_nelhage_rules_boost//:boost/boost.bzl", 60 | "boost_deps", 61 | ) 62 | 63 | boost_deps() 64 | 65 | git_repository( 66 | name = "com_google_absl", 67 | commit = "5e0dcf72c64fae912184d2e0de87195fe8f0a425", 68 | remote = "https://github.com/abseil/abseil-cpp", 69 | ) 70 | 71 | git_repository( 72 | name = "io_bazel_rules_go", 73 | commit = "da0ac6f8d552baef52abf8c36e669df306f416d0", # 0.18.1 74 | remote = "https://github.com/bazelbuild/rules_go.git", 75 | ) 76 | 77 | git_repository( 78 | name = "bazel_gazelle", 79 | commit = "e443c54b396a236e0d3823f46c6a931e1c9939f2", # 0.17.0 80 | remote = "https://github.com/bazelbuild/bazel-gazelle.git", 81 | ) 82 | 83 | load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") 84 | 85 | go_rules_dependencies() 86 | 87 | go_register_toolchains() 88 | 89 | load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") 90 | 91 | gazelle_dependencies() 92 | 93 | load( 94 | "//tools/build_defs:go_externals.bzl", 95 | "go_externals", 96 | ) 97 | 98 | go_externals() 99 | 100 | load( 101 | "//tools/build_defs:libgit2.bzl", 102 | "new_libgit2_archive", 103 | ) 104 | 105 | new_libgit2_archive( 106 | name = "com_github_libgit2", 107 | build_file = "//third_party:BUILD.libgit2", 108 | sha256 = "0269ec198c54e44f275f8f51e7391681a03aa45555e2ab6ce60b0757b6bde3de", 109 | url = "https://github.com/libgit2/libgit2/archive/v0.24.1.tar.gz", 110 | version = "0.24.1", 111 | ) 112 | 113 | load( 114 | "@build_stack_rules_proto//cpp:deps.bzl", 115 | "cpp_grpc_compile", 116 | "cpp_proto_compile", 117 | ) 118 | 119 | cpp_proto_compile() 120 | 121 | cpp_grpc_compile() 122 | 123 | load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") 124 | 125 | grpc_deps() 126 | 127 | git_repository( 128 | name = "io_bazel_buildifier", 129 | commit = "ae772d29d07002dfd89ed1d9ff673a1721f1b8dd", 130 | remote = "https://github.com/bazelbuild/buildifier.git", 131 | ) 132 | 133 | git_repository( 134 | name = "org_dropbox_rules_node", 135 | commit = "f35666fe95bff69b9c96b95a97973ee5bef108bd", 136 | remote = "https://github.com/dropbox/rules_node.git", 137 | ) 138 | 139 | load("@org_dropbox_rules_node//node:defs.bzl", "node_repositories") 140 | 141 | node_repositories() 142 | 143 | new_git_repository( 144 | name = "compdb", 145 | build_file_content = ( 146 | """ 147 | package(default_visibility = ["//visibility:public"]) 148 | """ 149 | ), 150 | commit = "7bc80f9355b09466fffabce24d463d65e37fcc0f", 151 | remote = "https://github.com/grailbio/bazel-compilation-database.git", 152 | ) 153 | 154 | git_repository( 155 | name = "com_google_googletest", 156 | commit = "0ea2d8f8fa1601abb9ce713b7414e7b86f90bc61", 157 | remote = "https://github.com/google/googletest", 158 | ) 159 | -------------------------------------------------------------------------------- /blameworthy/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "gitops.go", 7 | "indexer.go", 8 | ], 9 | importpath = "github.com/livegrep/livegrep/blameworthy", 10 | visibility = ["//visibility:public"], 11 | ) 12 | 13 | go_test( 14 | name = "go_default_test", 15 | size = "small", 16 | srcs = [ 17 | "gitops_test.go", 18 | "indexer_test.go", 19 | ], 20 | data = glob(["test_data/*"]), 21 | embed = [":go_default_library"], 22 | importpath = "github.com/livegrep/livegrep/blameworthy", 23 | ) 24 | -------------------------------------------------------------------------------- /blameworthy/gitops_test.go: -------------------------------------------------------------------------------- 1 | package blameworthy 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func TestLogParsing(t *testing.T) { 11 | test_log_parsing_file(t, "test_data/git-log.dashing") 12 | test_log_parsing_file(t, "test_data/git-log.dashing.stripped") 13 | } 14 | 15 | func test_log_parsing_file(t *testing.T, path string) { 16 | file, err := os.Open(path) 17 | if err != nil { 18 | t.Error(err) 19 | } 20 | defer file.Close() 21 | history, err := ParseGitLog(file) 22 | a := []string{} 23 | for k, diffs := range history.Files { 24 | a = append(a, fmt.Sprint(k, " -> ")) 25 | for _, d := range diffs { 26 | a = append(a, fmt.Sprintf("{%v %v %v %d}", 27 | d.Commit.Hash, d.Path, d.Hunks, d.LineCountAfter)) 28 | } 29 | } 30 | actual := strings.Join(a, "") 31 | wanted := "test.txt -> " + 32 | "{b9a26a4383eb51c1 test.txt [{0 0 1 3}] 3}" + 33 | "{b0539826eadc3feb test.txt [{1 2 1 2}] 3}" + 34 | "{42838bca4ba13c3f test.txt [{1 3 0 0}] 0}" 35 | if actual != wanted { 36 | t.Fatalf( 37 | "Git log parsed incorrectly\nWanted: %v\nActual: %v", 38 | wanted, actual, 39 | ) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /blameworthy/test_data/git-log.dashing: -------------------------------------------------------------------------------- 1 | commit b9a26a4383eb51c15701f57b27356071cf8c6c61 2 | 3 | diff --git test.txt test.txt 4 | new file mode 100644 5 | index 0000000..f6c64ed 6 | --- /dev/null 7 | +++ test.txt 8 | @@ -0,0 +1,3 @@ 9 | +-- These lines are designed 10 | +-- to confuse a reader 11 | +-- that thinks '-- ' is special 12 | commit b0539826eadc3febd8dd6ed962aee218d0b14fa2 13 | 14 | diff --git test.txt test.txt 15 | index f6c64ed..0b0d754 100644 16 | --- test.txt 17 | +++ test.txt 18 | @@ -1,2 +1,2 @@ 19 | --- These lines are designed 20 | --- to confuse a reader 21 | +-- These lines are specially designed 22 | +-- to confuse a reader of git diff output 23 | commit 42838bca4ba13c3f951854906845019b853ca4ad 24 | 25 | diff --git test.txt test.txt 26 | index 0b0d754..e69de29 100644 27 | --- test.txt 28 | +++ test.txt 29 | @@ -1,3 +0,0 @@ 30 | --- These lines are specially designed 31 | --- to confuse a reader of git diff output 32 | --- that thinks '-- ' is special 33 | -------------------------------------------------------------------------------- /blameworthy/test_data/git-log.dashing.stripped: -------------------------------------------------------------------------------- 1 | commit b9a26a4383eb51c15701f57b27356071cf8c6c61 2 | 3 | diff --git test.txt test.txt 4 | new file mode 100644 5 | index 0000000..f6c64ed 6 | --- /dev/null 7 | +++ test.txt 8 | @@ -0,0 +1,3 @@- 9 | commit b0539826eadc3febd8dd6ed962aee218d0b14fa2 10 | 11 | diff --git test.txt test.txt 12 | index f6c64ed..0b0d754 100644 13 | --- test.txt 14 | +++ test.txt 15 | @@ -1,2 +1,2 @@- 16 | commit 42838bca4ba13c3f951854906845019b853ca4ad 17 | 18 | diff --git test.txt test.txt 19 | index 0b0d754..e69de29 100644 20 | --- test.txt 21 | +++ test.txt 22 | @@ -1,3 +0,0 @@- 23 | -------------------------------------------------------------------------------- /client/test/BUILD: -------------------------------------------------------------------------------- 1 | # gazelle:ignore 2 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 3 | 4 | go_library( 5 | name = "go_default_library", 6 | srcs = ["testutil.go"], 7 | importpath = "github.com/livegrep/livegrep/client", 8 | visibility = ["//visibility:public"], 9 | deps = [ 10 | "//src/proto:go_proto", 11 | "@org_golang_google_grpc//:go_default_library", 12 | ], 13 | ) 14 | 15 | go_test( 16 | name = "go_default_test", 17 | srcs = [ 18 | "bench_test.go", 19 | "integration_test.go", 20 | ], 21 | embed = [":go_default_library"], 22 | deps = [ 23 | "//src/proto:go_proto", 24 | "@in_gopkg_check_v1//:go_default_library", 25 | ], 26 | ) 27 | -------------------------------------------------------------------------------- /client/test/bench_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "testing" 7 | 8 | pb "github.com/livegrep/livegrep/src/proto/go_proto" 9 | ) 10 | 11 | var index = flag.String("index", "", "Path to an index to run benchmarks against") 12 | 13 | func benchmarkQuery(b *testing.B, q *pb.Query) { 14 | if *index == "" { 15 | b.SkipNow() 16 | } 17 | 18 | c, e := NewClient("-load_index", *index) 19 | if e != nil { 20 | b.Fatal(e.Error()) 21 | } 22 | 23 | b.ResetTimer() 24 | for i := 0; i < b.N+1; i++ { 25 | _, e := c.Search(context.Background(), q) 26 | if e != nil { 27 | b.Fatalf("query: %s", e.Error()) 28 | } 29 | } 30 | } 31 | 32 | func BenchmarkDazed(b *testing.B) { 33 | benchmarkQuery(b, &pb.Query{Line: `dazed`}) 34 | } 35 | 36 | func BenchmarkDazedCaseFold(b *testing.B) { 37 | benchmarkQuery(b, &pb.Query{Line: `dazed`, FoldCase: true}) 38 | } 39 | 40 | func BenchmarkDefKmalloc(b *testing.B) { 41 | benchmarkQuery(b, &pb.Query{Line: `^(\s.*\S)?kmalloc\s*\(`}) 42 | } 43 | 44 | func BenchmarkSpaceEOL(b *testing.B) { 45 | benchmarkQuery(b, &pb.Query{Line: `\s$`}) 46 | } 47 | 48 | func Benchmark10Space(b *testing.B) { 49 | benchmarkQuery(b, &pb.Query{Line: `\s{10}$`}) 50 | } 51 | 52 | func BenchmarkUUID(b *testing.B) { 53 | benchmarkQuery(b, &pb.Query{ 54 | Line: `[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`, 55 | }) 56 | } 57 | 58 | func BenchmarkUUIDCaseFold(b *testing.B) { 59 | benchmarkQuery(b, &pb.Query{ 60 | Line: `[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`, 61 | FoldCase: true, 62 | }) 63 | } 64 | 65 | func BenchmarkAlphanum20(b *testing.B) { 66 | benchmarkQuery(b, &pb.Query{Line: `[0-9a-f]{20}`}) 67 | } 68 | 69 | func BenchmarkAlphanum50(b *testing.B) { 70 | benchmarkQuery(b, &pb.Query{Line: `[0-9a-f]{50}`}) 71 | } 72 | 73 | func BenchmarkAlphanum100(b *testing.B) { 74 | benchmarkQuery(b, &pb.Query{Line: `[0-9a-f]{50}`}) 75 | } 76 | -------------------------------------------------------------------------------- /client/test/patterns: -------------------------------------------------------------------------------- 1 | hello 2 | {10} 3 | ^(\s.*\S)?printk\s*\( 4 | (set_fs|interrupt) 5 | -------------------------------------------------------------------------------- /client/test/suites/basic: -------------------------------------------------------------------------------- 1 | ____do 2 | errno\W 3 | kmalloc\( 4 | printk\( 5 | ^(\s.*\S)?kmalloc\s*\( 6 | ^(\s.*\S)?printk\s*\( 7 | ^(\s.*\S)?acct_ 8 | dazed 9 | .zqz 10 | \s$ 11 | \s{10} 12 | \s<> 13 | \s{3}<> 14 | h(al|e*)t 15 | hello{0} 16 | -------------------------------------------------------------------------------- /client/test/suites/benchmarks: -------------------------------------------------------------------------------- 1 | [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} 2 | [0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12} 3 | [0-9a-fA-F]{20} 4 | [0-9a-fA-F]{50} 5 | [0-9a-f]{20} 6 | [0-9a-f]{50} 7 | [0-9a-f]{100} 8 | \s[0-9a-f]{199} 9 | [0-9a-fA-F]{200} 10 | -------------------------------------------------------------------------------- /client/test/suites/slow: -------------------------------------------------------------------------------- 1 | [0-9a-fA-F]{200} 2 | [0-9a-f]{50} 3 | [0-9a-f]{100} 4 | \s[0-9a-f]{199} 5 | [0-9a-fA-F]{200} 6 | a.*b.*c.*d.*e.*f.*g.*h.*i.*j.*k.*l.*m.*n.*o.*p.*q.*r.*s.*s 7 | -------------------------------------------------------------------------------- /client/test/suites/testcases: -------------------------------------------------------------------------------- 1 | 2 | ____do 3 | errno\W 4 | kmalloc\( 5 | printk\( 6 | ^(\s.*\S)?kmalloc\s*\( 7 | ^(\s.*\S)?printk\s*\( 8 | ^(\s.*\S)?acct_ 9 | . 10 | ^$ 11 | \ba*u*r*g+h+\b 12 | dazed 13 | .zqz 14 | \s$ 15 | \s{10} 16 | \s<> 17 | \s{3}<> 18 | h(al|e*)t 19 | hello{0} 20 | [Hh]ello World 21 | [wW][tT][fF] 22 | -------------------------------------------------------------------------------- /client/test/suites/unicode: -------------------------------------------------------------------------------- 1 | [À-ÿ] 2 | [À-Ð]{2} 3 | [0-9À-Ð]{4} 4 | -------------------------------------------------------------------------------- /client/test/testutil.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | 9 | "google.golang.org/grpc" 10 | 11 | pb "github.com/livegrep/livegrep/src/proto/go_proto" 12 | ) 13 | 14 | type TestClient struct { 15 | client pb.CodeSearchClient 16 | cmd *exec.Cmd 17 | wait chan error 18 | } 19 | 20 | func (c *TestClient) Info(ctx context.Context, in *pb.InfoRequest, opts ...grpc.CallOption) (*pb.ServerInfo, error) { 21 | return c.client.Info(ctx, in, opts...) 22 | } 23 | 24 | func (c *TestClient) Search(ctx context.Context, in *pb.Query, opts ...grpc.CallOption) (*pb.CodeSearchResult, error) { 25 | return c.client.Search(ctx, in, opts...) 26 | } 27 | 28 | func (c *TestClient) Close() { 29 | c.cmd.Process.Kill() 30 | <-c.wait 31 | } 32 | 33 | const Codesearch = "../../bazel-bin/src/tools/codesearch" 34 | const Port = 9812 35 | 36 | func NewClient(args ...string) (*TestClient, error) { 37 | addr := fmt.Sprintf("localhost:%d", Port) 38 | args = append([]string{"-grpc", addr}, 39 | args...) 40 | cl := &TestClient{wait: make(chan error)} 41 | cl.cmd = exec.Command(Codesearch, args...) 42 | 43 | cl.cmd.Stderr = os.Stderr 44 | 45 | if e := cl.cmd.Start(); e != nil { 46 | return nil, e 47 | } 48 | 49 | go func() { 50 | cl.wait <- cl.cmd.Wait() 51 | }() 52 | 53 | conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithBlock()) 54 | if err != nil { 55 | cl.Close() 56 | return nil, err 57 | } 58 | 59 | cl.client = pb.NewCodeSearchClient(conn) 60 | 61 | return cl, nil 62 | } 63 | -------------------------------------------------------------------------------- /cmd/BUILD: -------------------------------------------------------------------------------- 1 | load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") 2 | 3 | pkg_tar( 4 | name = "go_tools", 5 | srcs = [ 6 | "//cmd/{}:{}".format(cmd, cmd) 7 | for cmd in [ 8 | "lg", 9 | "livegrep", 10 | "livegrep-fetch-reindex", 11 | "livegrep-github-reindex", 12 | "livegrep-reload", 13 | ] 14 | ], 15 | package_dir = "/bin", 16 | visibility = ["//visibility:public"], 17 | ) 18 | -------------------------------------------------------------------------------- /cmd/lg/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["main.go"], 6 | importpath = "github.com/livegrep/livegrep/cmd/lg", 7 | visibility = ["//visibility:private"], 8 | deps = [ 9 | "//server/api:go_default_library", 10 | "@com_github_nelhage_go_cli//config:go_default_library", 11 | ], 12 | ) 13 | 14 | go_binary( 15 | name = "lg", 16 | embed = [":go_default_library"], 17 | visibility = ["//visibility:public"], 18 | ) 19 | -------------------------------------------------------------------------------- /cmd/lg/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "net" 8 | "net/http" 9 | "net/url" 10 | "os" 11 | "strings" 12 | 13 | "github.com/livegrep/livegrep/server/api" 14 | "github.com/nelhage/go.cli/config" 15 | ) 16 | 17 | var ( 18 | server = flag.String("server", "http://localhost:8910", "The livegrep server to connect to") 19 | unixSocket = flag.String("unix_socket", "", "unix socket path to connect() to as a proxy") 20 | showVersion = flag.Bool("show_version", false, "Show versions of matched packages") 21 | ) 22 | 23 | func main() { 24 | flag.Usage = func() { 25 | fmt.Fprintf(os.Stderr, "Usage: %s [flags] REGEX\n", os.Args[0]) 26 | flag.PrintDefaults() 27 | } 28 | if err := config.LoadConfig(flag.CommandLine, "lgrc"); err != nil { 29 | fmt.Fprintf(os.Stderr, "Loading config: %s\n", err) 30 | os.Exit(1) 31 | } 32 | flag.Parse() 33 | 34 | if len(flag.Args()) == 0 { 35 | flag.Usage() 36 | os.Exit(1) 37 | } 38 | 39 | var uri *url.URL 40 | var err error 41 | 42 | if strings.Contains(*server, ":") { 43 | if uri, err = url.Parse(*server); err != nil { 44 | fmt.Fprintf(os.Stderr, "Parsing server %s: %s\n", *server, err.Error()) 45 | os.Exit(1) 46 | } 47 | } else { 48 | uri = &url.URL{Scheme: "http", Host: *server} 49 | } 50 | 51 | uri.Path = "/api/v1/search/" 52 | uri.RawQuery = url.Values{"q": []string{strings.Join(flag.Args(), " ")}}.Encode() 53 | 54 | var transport http.RoundTripper 55 | if *unixSocket == "" { 56 | transport = http.DefaultTransport 57 | } else { 58 | dialUnix := func(network, addr string) (net.Conn, error) { 59 | return net.Dial("unix", *unixSocket) 60 | } 61 | transport = &http.Transport{ 62 | Dial: dialUnix, 63 | DialTLS: dialUnix, 64 | DisableKeepAlives: true, 65 | } 66 | } 67 | client := http.Client{Transport: transport} 68 | 69 | resp, err := client.Get(uri.String()) 70 | 71 | if err != nil { 72 | fmt.Fprintf(os.Stderr, "Requesting %s: %s\n", uri.String(), err.Error()) 73 | os.Exit(1) 74 | } 75 | 76 | if resp.StatusCode != 200 { 77 | var reply api.ReplyError 78 | if e := json.NewDecoder(resp.Body).Decode(&reply); e != nil { 79 | fmt.Fprintf(os.Stderr, 80 | "Error reading reply (status=%d): %s\n", resp.StatusCode, e.Error()) 81 | } else { 82 | fmt.Fprintf(os.Stderr, "Error: %s: %s\n", reply.Err.Code, reply.Err.Message) 83 | } 84 | os.Exit(1) 85 | } 86 | 87 | var reply api.ReplySearch 88 | 89 | if e := json.NewDecoder(resp.Body).Decode(&reply); e != nil { 90 | fmt.Fprintf(os.Stderr, 91 | "Error reading reply (status=%d): %s\n", resp.StatusCode, e.Error()) 92 | os.Exit(1) 93 | } 94 | 95 | for _, r := range reply.Results { 96 | if r.Tree != "" { 97 | fmt.Printf("%s:", r.Tree) 98 | } 99 | if *showVersion && r.Version != "" { 100 | fmt.Printf("%s:", r.Version) 101 | } 102 | fmt.Printf("%s:%d: ", r.Path, r.LineNumber) 103 | fmt.Printf("%s\n", r.Line) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /cmd/livegrep-fetch-reindex/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["main.go"], 6 | data = [ 7 | "//src/tools:codesearch", 8 | ], 9 | importpath = "github.com/livegrep/livegrep/cmd/livegrep-fetch-reindex", 10 | visibility = ["//visibility:private"], 11 | deps = [ 12 | "//src/proto:go_proto", 13 | "@org_golang_google_grpc//:go_default_library", 14 | ], 15 | ) 16 | 17 | go_binary( 18 | name = "livegrep-fetch-reindex", 19 | embed = [":go_default_library"], 20 | visibility = ["//visibility:public"], 21 | ) 22 | -------------------------------------------------------------------------------- /cmd/livegrep-fetch-reindex/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "os" 11 | "os/exec" 12 | "path" 13 | "strings" 14 | "sync" 15 | 16 | pb "github.com/livegrep/livegrep/src/proto/go_proto" 17 | "google.golang.org/grpc" 18 | ) 19 | 20 | type IndexConfig struct { 21 | Name string `json:"name"` 22 | Repositories []RepoConfig `json:"repositories"` 23 | } 24 | 25 | type RepoConfig struct { 26 | Path string `json:"path"` 27 | Name string `json:"name"` 28 | Revisions []string `json:"revisions"` 29 | Metadata map[string]string `json:"metadata"` 30 | } 31 | 32 | var ( 33 | flagCodesearch = flag.String("codesearch", path.Join(path.Dir(os.Args[0]), "codesearch"), "Path to the `codesearch` binary") 34 | flagIndexPath = flag.String("out", "livegrep.idx", "Path to write the index") 35 | flagRevparse = flag.Bool("revparse", true, "whether to `git rev-parse` the provided revision in generated links") 36 | flagSkipMissing = flag.Bool("skip-missing", false, "skip repositories where the specified revision is missing") 37 | flagReloadBackend = flag.String("reload-backend", "", "Backend to send a Reload RPC to") 38 | ) 39 | 40 | const Workers = 8 41 | 42 | func main() { 43 | flag.Parse() 44 | log.SetFlags(0) 45 | 46 | if len(flag.Args()) != 1 { 47 | log.Fatal("Expected exactly one argument (the index json configuration)") 48 | } 49 | 50 | data, err := ioutil.ReadFile(flag.Arg(0)) 51 | if err != nil { 52 | log.Fatalf(err.Error()) 53 | } 54 | 55 | var cfg IndexConfig 56 | if err = json.Unmarshal(data, &cfg); err != nil { 57 | log.Fatalf("reading %s: %s", flag.Arg(0), err.Error()) 58 | } 59 | 60 | if err := checkoutRepos(&cfg.Repositories); err != nil { 61 | log.Fatalln(err.Error()) 62 | } 63 | 64 | tmp := *flagIndexPath + ".tmp" 65 | 66 | args := []string{ 67 | "--debug=ui", 68 | "--dump_index", 69 | tmp, 70 | "--index_only", 71 | } 72 | if *flagRevparse { 73 | args = append(args, "--revparse") 74 | } 75 | args = append(args, flag.Arg(0)) 76 | 77 | cmd := exec.Command(*flagCodesearch, args...) 78 | cmd.Stdout = os.Stdout 79 | cmd.Stderr = os.Stderr 80 | if err := cmd.Run(); err != nil { 81 | log.Fatalln(err) 82 | } 83 | 84 | if err := os.Rename(tmp, *flagIndexPath); err != nil { 85 | log.Fatalln("rename:", err.Error()) 86 | } 87 | 88 | if *flagReloadBackend != "" { 89 | if err := reloadBackend(*flagReloadBackend); err != nil { 90 | log.Fatalln("reload:", err.Error()) 91 | } 92 | } 93 | } 94 | 95 | func checkoutRepos(repos *[]RepoConfig) error { 96 | repoc := make(chan *RepoConfig) 97 | errc := make(chan error, Workers) 98 | stop := make(chan struct{}) 99 | wg := sync.WaitGroup{} 100 | wg.Add(Workers) 101 | for i := 0; i < Workers; i++ { 102 | go func() { 103 | defer wg.Done() 104 | checkoutWorker(repoc, stop, errc) 105 | }() 106 | } 107 | 108 | var err error 109 | Repos: 110 | for i := range *repos { 111 | select { 112 | case repoc <- &(*repos)[i]: 113 | case err = <-errc: 114 | close(stop) 115 | break Repos 116 | } 117 | } 118 | 119 | close(repoc) 120 | wg.Wait() 121 | select { 122 | case err = <-errc: 123 | default: 124 | } 125 | 126 | return err 127 | } 128 | 129 | func checkoutWorker(c <-chan *RepoConfig, 130 | stop <-chan struct{}, errc chan error) { 131 | for { 132 | select { 133 | case r, ok := <-c: 134 | if !ok { 135 | return 136 | } 137 | err := checkoutOne(r) 138 | if err != nil { 139 | errc <- err 140 | } 141 | case <-stop: 142 | return 143 | } 144 | } 145 | } 146 | 147 | func retryCommand(program string, args []string) error { 148 | var err error 149 | for i := 0; i < 3; i++ { 150 | cmd := exec.Command("git", args...) 151 | cmd.Stdout = os.Stdout 152 | cmd.Stderr = os.Stderr 153 | if err = cmd.Run(); err == nil { 154 | return nil 155 | } 156 | } 157 | return fmt.Errorf("%s %v: %s", program, args, err.Error()) 158 | } 159 | 160 | func checkoutOne(r *RepoConfig) error { 161 | log.Println("Updating", r.Name) 162 | 163 | remote, ok := r.Metadata["remote"] 164 | if !ok { 165 | return fmt.Errorf("git remote not found in repository metadata for %s", r.Name) 166 | } 167 | 168 | out, err := exec.Command("git", "-C", r.Path, "rev-parse", "--is-bare-repository").Output() 169 | if err != nil { 170 | if _, ok := err.(*exec.ExitError); !ok { 171 | return err 172 | } 173 | } 174 | if strings.Trim(string(out), " \n") != "true" { 175 | if err := os.RemoveAll(r.Path); err != nil { 176 | return err 177 | } 178 | if err := os.MkdirAll(r.Path, 0755); err != nil { 179 | return err 180 | } 181 | return retryCommand("git", []string{"clone", "--mirror", remote, r.Path}) 182 | } 183 | 184 | if err := exec.Command("git", "-C", r.Path, "remote", "set-url", "origin", remote).Run(); err != nil { 185 | return err 186 | } 187 | 188 | return retryCommand("git", []string{"-C", r.Path, "fetch", "-p", "origin"}) 189 | } 190 | 191 | func reloadBackend(addr string) error { 192 | client, err := grpc.Dial(addr, grpc.WithInsecure()) 193 | if err != nil { 194 | return err 195 | } 196 | 197 | codesearch := pb.NewCodeSearchClient(client) 198 | 199 | if _, err = codesearch.Reload(context.Background(), &pb.Empty{}, grpc.FailFast(false)); err != nil { 200 | return err 201 | } 202 | return nil 203 | } 204 | -------------------------------------------------------------------------------- /cmd/livegrep-git-log/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "main.go", 7 | ], 8 | importpath = "github.com/livegrep/livegrep/cmd/livegrep-git-log", 9 | visibility = ["//visibility:public"], 10 | deps = ["//blameworthy:go_default_library"], 11 | ) 12 | 13 | go_binary( 14 | name = "livegrep-git-log", 15 | embed = [":go_default_library"], 16 | importpath = "github.com/livegrep/livegrep/cmd/livegrep-git-log", 17 | visibility = ["//visibility:public"], 18 | ) 19 | -------------------------------------------------------------------------------- /cmd/livegrep-git-log/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | This package is a utility that strips diff content from a git log. 3 | 4 | This small utility reads a git log from standard input, and writes it to 5 | standard output preserving only four kinds of line: 6 | 7 | "commit ..." <- names the commit 8 | "--- ..." <- at the top of each file 9 | "+++ ..." <- at the top of each file 10 | "@@ ..." <- at the start of each hunk 11 | 12 | It edits each "@@ -0,0 +1,3 @@" line so its second "@@" is followed by a 13 | dash instead of a space ("@@-", which never happens in real diffs) so 14 | our blame input routine knows to not expect "+" and "-" lines to follow. 15 | This makes the log far more compact, so livegrep can scan it more 16 | quickly during startup. 17 | 18 | */ 19 | package main 20 | 21 | import ( 22 | "log" 23 | "os" 24 | 25 | "github.com/livegrep/livegrep/blameworthy" 26 | ) 27 | 28 | func main() { 29 | target := "HEAD" 30 | if len(os.Args) == 3 { 31 | target = os.Args[2] 32 | } else if len(os.Args) != 2 { 33 | log.Fatalf("usage: %s []") 34 | } 35 | input, err := blameworthy.RunGitLog(os.Args[1], target) 36 | if err != nil { 37 | log.Fatal(err) 38 | } 39 | err = blameworthy.StripGitLog(input) 40 | if err != nil { 41 | log.Fatal(err) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /cmd/livegrep-github-reindex/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "flags.go", 7 | "main.go", 8 | ], 9 | importpath = "github.com/livegrep/livegrep/cmd/livegrep-github-reindex", 10 | visibility = ["//visibility:private"], 11 | deps = [ 12 | "@com_github_google_go_github//github:go_default_library", 13 | "@org_golang_x_net//context:go_default_library", 14 | "@org_golang_x_oauth2//:go_default_library", 15 | ], 16 | ) 17 | 18 | go_binary( 19 | name = "livegrep-github-reindex", 20 | embed = [":go_default_library"], 21 | visibility = ["//visibility:public"], 22 | ) 23 | -------------------------------------------------------------------------------- /cmd/livegrep-github-reindex/flags.go: -------------------------------------------------------------------------------- 1 | // Implement some custom flag.Value instances for use in main.go 2 | package main 3 | 4 | import "strings" 5 | 6 | type stringList struct { 7 | strings []string 8 | } 9 | 10 | func (s *stringList) String() string { 11 | return strings.Join(s.strings, ", ") 12 | } 13 | 14 | func (s *stringList) Set(str string) error { 15 | s.strings = append(s.strings, str) 16 | return nil 17 | } 18 | 19 | func (s *stringList) Get() interface{} { 20 | return s.strings 21 | } 22 | 23 | type dynamicDefault struct { 24 | val string 25 | display string 26 | fn func() string 27 | } 28 | 29 | func (d *dynamicDefault) String() string { 30 | if d.val != "" { 31 | return d.val 32 | } 33 | return d.display 34 | } 35 | 36 | func (d *dynamicDefault) Get() interface{} { 37 | if d.val != "" { 38 | return d.val 39 | } 40 | return d.fn() 41 | } 42 | 43 | func (d *dynamicDefault) Set(str string) error { 44 | d.val = str 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /cmd/livegrep-reload/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["main.go"], 6 | data = [ 7 | "//src/tools:codesearch", 8 | ], 9 | importpath = "github.com/livegrep/livegrep/cmd/livegrep-reload", 10 | visibility = ["//visibility:private"], 11 | deps = [ 12 | "//src/proto:go_proto", 13 | "@org_golang_google_grpc//:go_default_library", 14 | ], 15 | ) 16 | 17 | go_binary( 18 | name = "livegrep-reload", 19 | embed = [":go_default_library"], 20 | visibility = ["//visibility:public"], 21 | ) 22 | -------------------------------------------------------------------------------- /cmd/livegrep-reload/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "log" 7 | 8 | pb "github.com/livegrep/livegrep/src/proto/go_proto" 9 | "google.golang.org/grpc" 10 | ) 11 | 12 | func main() { 13 | flag.Parse() 14 | log.SetFlags(0) 15 | 16 | if len(flag.Args()) != 1 { 17 | log.Fatal("You must provide a HOST:PORT to reload") 18 | } 19 | 20 | if err := reloadBackend(flag.Arg(0)); err != nil { 21 | log.Fatalln("reload:", err.Error()) 22 | } 23 | } 24 | 25 | func reloadBackend(addr string) error { 26 | client, err := grpc.Dial(addr, grpc.WithInsecure()) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | codesearch := pb.NewCodeSearchClient(client) 32 | 33 | if _, err = codesearch.Reload(context.Background(), &pb.Empty{}, grpc.FailFast(true)); err != nil { 34 | return err 35 | } 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /cmd/livegrep-update-blame-cache/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "main.go", 7 | ], 8 | importpath = "github.com/livegrep/livegrep/cmd/livegrep-update-blame-cache", 9 | visibility = ["//visibility:public"], 10 | deps = ["//blameworthy:go_default_library"], 11 | ) 12 | 13 | go_binary( 14 | name = "livegrep-update-blame-cache", 15 | embed = [":go_default_library"], 16 | importpath = "github.com/livegrep/livegrep/cmd/livegrep-update-blame-cache", 17 | visibility = ["//visibility:public"], 18 | ) 19 | -------------------------------------------------------------------------------- /cmd/livegrep-update-blame-cache/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | // "flag" 5 | "fmt" 6 | "log" 7 | "os" 8 | "time" 9 | 10 | "github.com/livegrep/livegrep/blameworthy" 11 | ) 12 | 13 | func main() { 14 | // flag.Parse() 15 | // log.SetFlags(0) 16 | file, err := os.Open("/home/brhodes/log3.server") 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | defer file.Close() 21 | 22 | start := time.Now() 23 | histories, _ := blameworthy.ParseGitLog(file) 24 | elapsed := time.Since(start) 25 | log.Printf("Git log loaded in %s", elapsed) 26 | 27 | // fmt.Printf("%d commits\n", len(*commits)) 28 | 29 | fmt.Printf("%d commits\n", len(histories.Commits)) 30 | fmt.Printf("%d files\n", len(histories.Files)) 31 | 32 | // Which file has the longest history? 33 | 34 | // Which file had the most lines changed? 35 | 36 | // Which file had the most expensive history? 37 | // file_lengths := make(map[string]int) 38 | // m := make(map[string]int) 39 | // for _, commit := range *commits { 40 | // for _, file := range commit.Files { 41 | // for _, h := range file.Hunks { 42 | // file_lengths[file.Path] -= h.Old_length 43 | // file_lengths[file.Path] += h.New_length 44 | // m[file.Path] += file_lengths[file.Path] 45 | // } 46 | // } 47 | // } 48 | 49 | // f, err := os.Create("/home/brhodes/tmp.out") 50 | // for k, v := range m { 51 | // fmt.Fprintf(f, "%d %s\n", v, k) 52 | // } 53 | 54 | target_path := "quarantine.txt" 55 | //target_path = "dropbox/api/v2/datatypes/team_log.py" 56 | 57 | small_history := histories.Files[target_path] 58 | 59 | fmt.Printf("history length: %d\n", len(small_history)) 60 | 61 | // start = time.Now() 62 | // small_history.FileBlame(len(small_history) - 2) 63 | // elapsed = time.Since(start) 64 | 65 | // log.Printf("Small history loaded in %s", elapsed) 66 | 67 | // start = time.Now() 68 | // //i := 0b159401a6ebde40093ac8ace25944e81f4d3836 69 | // i := 1 70 | // blameVector, futureVector := small_history.DiffBlame(i) 71 | // elapsed = time.Since(start) 72 | 73 | // log.Printf("Diff history loaded in %s", elapsed) 74 | // log.Print(blameVector, "\n") 75 | // log.Print(futureVector, "\n") 76 | 77 | //time.Sleep(10 * time.Second) 78 | 79 | //blame_index := 80 | //blameworthy.Build_index(commits) 81 | //fmt.Print((*blame_index)["8e18c6e7:README.md"]) 82 | } 83 | -------------------------------------------------------------------------------- /cmd/livegrep/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["livegrep.go"], 6 | importpath = "github.com/livegrep/livegrep/cmd/livegrep", 7 | visibility = ["//visibility:private"], 8 | deps = [ 9 | "//server:go_default_library", 10 | "//server/config:go_default_library", 11 | "//server/middleware:go_default_library", 12 | "@com_github_honeycombio_libhoney_go//:go_default_library", 13 | ], 14 | ) 15 | 16 | go_binary( 17 | name = "livegrep", 18 | embed = [":go_default_library"], 19 | visibility = ["//visibility:public"], 20 | ) 21 | -------------------------------------------------------------------------------- /cmd/livegrep/livegrep.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | _ "expvar" 6 | "flag" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | _ "net/http/pprof" 11 | "os" 12 | "path" 13 | 14 | libhoney "github.com/honeycombio/libhoney-go" 15 | "github.com/livegrep/livegrep/server" 16 | "github.com/livegrep/livegrep/server/config" 17 | "github.com/livegrep/livegrep/server/middleware" 18 | ) 19 | 20 | var ( 21 | serveAddr = flag.String("listen", "127.0.0.1:8910", "The address to listen on") 22 | backendAddr = flag.String("connect", "localhost:9999", "The address to connect to") 23 | docRoot = flag.String("docroot", "", "The livegrep document root (web/ directory). If not provided, this defaults to web/ inside the bazel-created runfiles directory adjacent to the livegrep binary.") 24 | indexConfig = flag.String("index-config", "", "Codesearch index config file; provide to enable repo browsing") 25 | reload = flag.Bool("reload", false, "Reload template files on every request") 26 | _ = flag.Bool("logtostderr", false, "[DEPRECATED] compatibility with glog") 27 | ) 28 | 29 | func runfilesPath(sourcePath string) (string, error) { 30 | programPath, err := os.Executable() 31 | if err != nil { 32 | return "", err 33 | } 34 | return path.Join(programPath+".runfiles", "com_github_livegrep_livegrep", sourcePath), nil 35 | } 36 | 37 | func main() { 38 | flag.Parse() 39 | 40 | if *docRoot == "" { 41 | var err error 42 | *docRoot, err = runfilesPath("web") 43 | if err != nil { 44 | log.Fatalf(err.Error()) 45 | } 46 | } 47 | 48 | cfg := &config.Config{ 49 | DocRoot: *docRoot, 50 | Listen: *serveAddr, 51 | Reload: *reload, 52 | Backends: []config.Backend{ 53 | {Id: "", Addr: *backendAddr}, 54 | }, 55 | Honeycomb: config.Honeycomb{ 56 | WriteKey: os.Getenv("HONEYCOMB_WRITE_KEY"), 57 | Dataset: os.Getenv("HONEYCOMB_DATASET"), 58 | }, 59 | } 60 | 61 | if *indexConfig != "" { 62 | data, err := ioutil.ReadFile(*indexConfig) 63 | if err != nil { 64 | log.Fatalf(err.Error()) 65 | } 66 | 67 | if err = json.Unmarshal(data, &cfg.IndexConfig); err != nil { 68 | log.Fatalf("reading %s: %s", flag.Arg(0), err.Error()) 69 | } 70 | } 71 | 72 | if len(flag.Args()) != 0 { 73 | data, err := ioutil.ReadFile(flag.Arg(0)) 74 | if err != nil { 75 | log.Fatalf(err.Error()) 76 | } 77 | 78 | if err = json.Unmarshal(data, &cfg); err != nil { 79 | log.Fatalf("reading %s: %s", flag.Arg(0), err.Error()) 80 | } 81 | } 82 | 83 | libhoney.Init(libhoney.Config{}) 84 | 85 | handler, err := server.New(cfg) 86 | if err != nil { 87 | panic(err.Error()) 88 | } 89 | 90 | if cfg.ReverseProxy { 91 | handler = middleware.UnwrapProxyHeaders(handler) 92 | } 93 | 94 | http.DefaultServeMux.Handle("/", handler) 95 | 96 | log.Printf("Listening on %s.", cfg.Listen) 97 | log.Fatal(http.ListenAndServe(cfg.Listen, nil)) 98 | } 99 | -------------------------------------------------------------------------------- /doc/examples/livegrep/generate_ordered_contents.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # A simple example script for sorting paths in a directory to be indexed according to 4 | # their relevance. This can be used to help populate the ordered-contents field in 5 | # the fs_paths entry JSON. 6 | 7 | if [ -z "$1" ] 8 | then 9 | echo 'usage: generate_ordered_contents.sh DIRECTORY' 10 | exit 2 11 | fi 12 | 13 | cd "$1" 14 | 15 | find . -type f | awk ' 16 | {score = 100} 17 | /test/ {score -= 10} 18 | /BUILD/ {score -= 5} 19 | {print score, $0} 20 | ' | sort -k1nr | sed 's/^[-0-9]* //' 21 | -------------------------------------------------------------------------------- /doc/examples/livegrep/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "livegrep", 3 | "fs_paths": [ 4 | { 5 | "name": "livegrep/livegrep", 6 | "path": "src/", 7 | "metadata": { 8 | "url-pattern": "https://github.com/{name}/blob/HEAD/src/{path}#L{lno}" 9 | } 10 | } 11 | ], 12 | "repositories": [ 13 | { 14 | "name": "livegrep", 15 | "path": ".", 16 | "revisions": [ "HEAD" ], 17 | "metadata": { 18 | "github": "nelhage/livegrep" 19 | } 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /doc/examples/livegrep/index_with_ordered_contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "livegrep", 3 | "fs_paths": [ 4 | { 5 | "name": "livegrep/livegrep", 6 | "path": "src/", 7 | "ordered-contents": "ordered-contents.txt", 8 | "metadata": { 9 | "url-pattern": "https://github.com/{name}/blob/HEAD/src/{path}#L{lno}" 10 | } 11 | } 12 | ], 13 | } 14 | -------------------------------------------------------------------------------- /doc/examples/livegrep/ordered-contents.txt: -------------------------------------------------------------------------------- 1 | ./codesearch.cc 2 | ./codesearch.h 3 | ./BUILD 4 | -------------------------------------------------------------------------------- /doc/examples/livegrep/server.json: -------------------------------------------------------------------------------- 1 | { 2 | "backends": 3 | [ 4 | { 5 | "id": "livegrep", 6 | "addr": "localhost:9999" 7 | } 8 | ], 9 | "feedback": { 10 | "mailto": "nelhage@nelhage.com" 11 | }, 12 | "listen": "0.0.0.0:8910" 13 | } 14 | -------------------------------------------------------------------------------- /jsonframe/BUILD: -------------------------------------------------------------------------------- 1 | load( 2 | "@io_bazel_rules_go//go:def.bzl", 3 | "go_library", 4 | ) 5 | 6 | go_library( 7 | name = "go_default_library", 8 | srcs = ["jsonframe.go"], 9 | importpath = "github.com/livegrep/livegrep/jsonframe", 10 | visibility = ["//visibility:public"], 11 | ) 12 | -------------------------------------------------------------------------------- /jsonframe/jsonframe.go: -------------------------------------------------------------------------------- 1 | package jsonframe 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "reflect" 7 | ) 8 | 9 | type UnknownOpcode struct { 10 | Opcode string 11 | } 12 | 13 | func (u *UnknownOpcode) Error() string { 14 | return fmt.Sprintf("unknown opcode '%s'", u.Opcode) 15 | } 16 | 17 | type Frame struct { 18 | Opcode string `json:"opcode"` 19 | Body json.RawMessage `json:"body"` 20 | } 21 | 22 | type wFrame struct { 23 | Opcode string `json:"opcode"` 24 | Body interface{} `json:"body"` 25 | } 26 | 27 | type Op interface { 28 | Opcode() string 29 | } 30 | 31 | type Marshaler struct { 32 | ops map[string]Op 33 | } 34 | 35 | func (m *Marshaler) Register(o Op) { 36 | if m.ops == nil { 37 | m.ops = make(map[string]Op) 38 | } 39 | if _, ok := m.ops[o.Opcode()]; ok { 40 | panic(fmt.Sprintf("Register: duplicate opcode: %s", o.Opcode())) 41 | } 42 | m.ops[o.Opcode()] = o 43 | } 44 | 45 | func (m *Marshaler) Encode(e *json.Encoder, op Op) error { 46 | frame := &wFrame{op.Opcode(), op} 47 | return e.Encode(frame) 48 | } 49 | 50 | func (m *Marshaler) Marshal(op Op) ([]byte, error) { 51 | frame := &wFrame{op.Opcode(), op} 52 | return json.Marshal(frame) 53 | } 54 | 55 | func (m *Marshaler) unpack(f *Frame, out *Op) error { 56 | prototype, ok := m.ops[f.Opcode] 57 | if !ok { 58 | return &UnknownOpcode{f.Opcode} 59 | } 60 | 61 | op := reflect.New(reflect.TypeOf(prototype).Elem()).Interface().(Op) 62 | if err := json.Unmarshal(f.Body, op); err != nil { 63 | return err 64 | } 65 | 66 | *out = op 67 | return nil 68 | } 69 | 70 | func (m *Marshaler) Decode(d *json.Decoder) (Op, error) { 71 | var f Frame 72 | if err := d.Decode(&f); err != nil { 73 | return nil, err 74 | } 75 | var o Op 76 | if err := m.unpack(&f, &o); err != nil { 77 | return nil, err 78 | } 79 | return o, nil 80 | } 81 | 82 | func (m *Marshaler) Unmarshal(buf []byte, out *Op) error { 83 | var f Frame 84 | if err := json.Unmarshal(buf, &f); err != nil { 85 | return err 86 | } 87 | return m.unpack(&f, out) 88 | } 89 | -------------------------------------------------------------------------------- /package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | mkdir -p builds 4 | rev=$(git rev-parse HEAD | head -c10) 5 | builddir="livegrep-$rev" 6 | rm -rf "$builddir" && mkdir "$builddir" 7 | bazel build :livegrep 8 | tar -C "$builddir" -xf "$(bazel info bazel-bin)"/livegrep.tar 9 | tar -czf "builds/$builddir.tgz" "$builddir" 10 | rm -rf "$builddir" 11 | -------------------------------------------------------------------------------- /server/BUILD: -------------------------------------------------------------------------------- 1 | # gazelle:ignore 2 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 3 | 4 | go_library( 5 | name = "go_default_library", 6 | srcs = [ 7 | "api.go", 8 | "backend.go", 9 | "fastforward.go", 10 | "fileblame.go", 11 | "fileview.go", 12 | "json.go", 13 | "query.go", 14 | "server.go", 15 | ], 16 | data = [ 17 | "//web:asset_hashes", 18 | "//web:htdocs", 19 | "//web:templates", 20 | "//web:webpack_build", 21 | ], 22 | importpath = "github.com/livegrep/livegrep/server", 23 | visibility = ["//visibility:public"], 24 | deps = [ 25 | "//blameworthy:go_default_library", 26 | "//server/api:go_default_library", 27 | "//server/config:go_default_library", 28 | "//server/log:go_default_library", 29 | "//server/reqid:go_default_library", 30 | "//server/templates:go_default_library", 31 | "//src/proto:go_proto", 32 | "@com_github_bmizerany_pat//:go_default_library", 33 | "@com_github_honeycombio_libhoney_go//:go_default_library", 34 | "@org_golang_google_grpc//:go_default_library", 35 | "@org_golang_google_grpc//codes:go_default_library", 36 | "@org_golang_google_grpc//metadata:go_default_library", 37 | "@org_golang_x_net//context:go_default_library", 38 | ], 39 | ) 40 | 41 | go_test( 42 | name = "go_default_test", 43 | srcs = ["fastforward_test.go", "query_test.go", "server_test.go"], 44 | embed = [":go_default_library"], 45 | deps = ["//src/proto:go_proto"], 46 | ) 47 | -------------------------------------------------------------------------------- /server/api/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["types.go"], 6 | importpath = "github.com/livegrep/livegrep/server/api", 7 | visibility = ["//visibility:public"], 8 | ) 9 | -------------------------------------------------------------------------------- /server/api/types.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | type InnerError struct { 4 | Code string `json:"code"` 5 | Message string `json:"message"` 6 | } 7 | 8 | // ReplyError is returned along with any non-200 status reply 9 | type ReplyError struct { 10 | Err InnerError `json:"error"` 11 | } 12 | 13 | // ReplySearch is returned to /api/v1/search/:backend 14 | type ReplySearch struct { 15 | Info *Stats `json:"info"` 16 | Results []*Result `json:"results"` 17 | FileResults []*FileResult `json:"file_results"` 18 | SearchType string `json:"search_type"` 19 | } 20 | 21 | type Stats struct { 22 | RE2Time int64 `json:"re2_time"` 23 | GitTime int64 `json:"git_time"` 24 | SortTime int64 `json:"sort_time"` 25 | IndexTime int64 `json:"index_time"` 26 | AnalyzeTime int64 `json:"analyze_time"` 27 | TotalTime int64 `json:"total_time"` 28 | ExitReason string `json:"why"` 29 | } 30 | 31 | type Result struct { 32 | Tree string `json:"tree"` 33 | Version string `json:"version"` 34 | Path string `json:"path"` 35 | LineNumber int `json:"lno"` 36 | ContextBefore []string `json:"context_before"` 37 | ContextAfter []string `json:"context_after"` 38 | Bounds [2]int `json:"bounds"` 39 | Line string `json:"line"` 40 | } 41 | 42 | type FileResult struct { 43 | Tree string `json:"tree"` 44 | Version string `json:"version"` 45 | Path string `json:"path"` 46 | Bounds [2]int `json:"bounds"` 47 | } 48 | -------------------------------------------------------------------------------- /server/backend.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "net/url" 7 | "sync" 8 | "time" 9 | 10 | pb "github.com/livegrep/livegrep/src/proto/go_proto" 11 | "google.golang.org/grpc" 12 | ) 13 | 14 | type Tree struct { 15 | Name string 16 | Version string 17 | Url string 18 | } 19 | 20 | type I struct { 21 | Name string 22 | Trees []Tree 23 | sync.Mutex 24 | IndexTime time.Time 25 | } 26 | 27 | type Backend struct { 28 | Id string 29 | Addr string 30 | I *I 31 | Codesearch pb.CodeSearchClient 32 | } 33 | 34 | func NewBackend(id string, addr string) (*Backend, error) { 35 | client, err := grpc.Dial(addr, grpc.WithInsecure()) 36 | if err != nil { 37 | return nil, err 38 | } 39 | bk := &Backend{ 40 | Id: id, 41 | Addr: addr, 42 | I: &I{Name: id}, 43 | Codesearch: pb.NewCodeSearchClient(client), 44 | } 45 | return bk, nil 46 | } 47 | 48 | func (bk *Backend) Start() { 49 | if bk.I == nil { 50 | bk.I = &I{Name: bk.Id} 51 | } 52 | go bk.poll() 53 | } 54 | 55 | func (bk *Backend) poll() { 56 | for { 57 | info, e := bk.Codesearch.Info(context.Background(), &pb.InfoRequest{}, grpc.FailFast(false)) 58 | if e == nil { 59 | bk.refresh(info) 60 | } else { 61 | log.Printf("refresh %s: %v", bk.Id, e) 62 | } 63 | time.Sleep(60 * time.Second) 64 | } 65 | } 66 | 67 | func (bk *Backend) refresh(info *pb.ServerInfo) { 68 | bk.I.Lock() 69 | defer bk.I.Unlock() 70 | 71 | if info.Name != "" { 72 | bk.I.Name = info.Name 73 | } 74 | bk.I.IndexTime = time.Unix(info.IndexTime, 0) 75 | if len(info.Trees) > 0 { 76 | bk.I.Trees = nil 77 | for _, r := range info.Trees { 78 | pattern := r.Metadata.UrlPattern 79 | if v := r.Metadata.Github; v != "" { 80 | value := v 81 | base := "" 82 | _, err := url.ParseRequestURI(value) 83 | if err != nil { 84 | base = "https://github.com/" + value 85 | } else { 86 | base = value 87 | } 88 | pattern = base + "/blob/{version}/{path}#L{lno}" 89 | } 90 | bk.I.Trees = append(bk.I.Trees, 91 | Tree{r.Name, r.Version, pattern}) 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /server/config/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["config.go"], 6 | importpath = "github.com/livegrep/livegrep/server/config", 7 | visibility = ["//visibility:public"], 8 | ) 9 | -------------------------------------------------------------------------------- /server/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "html/template" 5 | ) 6 | 7 | type Backend struct { 8 | Id string `json:"id"` 9 | Addr string `json:"addr"` 10 | } 11 | 12 | type Honeycomb struct { 13 | WriteKey string `json:"write_key"` 14 | Dataset string `json:"dataset"` 15 | } 16 | 17 | type Config struct { 18 | // Location of the directory containing templates and static 19 | // assets. This should point at the "web" directory of the 20 | // repository. 21 | DocRoot string `json:"docroot"` 22 | 23 | Feedback struct { 24 | // The mailto address for the "feedback" url. 25 | MailTo string `json:"mailto"` 26 | } `json:"feedback"` 27 | 28 | GoogleAnalyticsId string `json:"google_analytics_id"` 29 | // Should we respect X-Real-Ip, X-Real-Proto, and X-Forwarded-Host? 30 | ReverseProxy bool `json:"reverse_proxy"` 31 | 32 | // List of backends to connect to. Each backend must include 33 | // the "id" and "addr" fields. 34 | Backends []Backend `json:"backends"` 35 | 36 | // The address to listen on, as HOST:PORT. 37 | Listen string `json:"listen"` 38 | 39 | // HTML injected into layout template 40 | // for site-specific customizations 41 | HeaderHTML template.HTML `json:"header_html"` 42 | 43 | Sentry struct { 44 | URI string `json:"uri"` 45 | } `json:"sentry"` 46 | 47 | // Whether to re-load templates on every request 48 | Reload bool `json:"reload"` 49 | 50 | // honeycomb API write key 51 | Honeycomb Honeycomb `json:"honeycomb"` 52 | 53 | DefaultMaxMatches int32 `json:"default_max_matches"` 54 | 55 | // Same json config structure that the backend uses when building indexes; 56 | // used here for repository browsing. 57 | IndexConfig IndexConfig `json:"index_config"` 58 | 59 | DefaultSearchRepos []string `json:"default_search_repos"` 60 | 61 | LinkConfigs []LinkConfig `json:"file_links"` 62 | } 63 | 64 | type IndexConfig struct { 65 | Name string `json:"name"` 66 | Repositories []RepoConfig `json:"repositories"` 67 | } 68 | 69 | type RepoConfig struct { 70 | Path string `json:"path"` 71 | Name string `json:"name"` 72 | Revisions []string `json:"revisions"` 73 | Metadata map[string]string `json:"metadata"` 74 | } 75 | 76 | type LinkConfig struct { 77 | Label string `json:"label"` 78 | UrlTemplate string `json:"url_template"` 79 | WhitelistPattern string `json:"whitelist_pattern"` 80 | } 81 | -------------------------------------------------------------------------------- /server/fastforward_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "testing" 7 | ) 8 | 9 | func TestAnalyzeEditAndMapLine(t *testing.T) { 10 | lines0 := []string{ 11 | "func my_function(arg1 int, arg2 int, arg3 string) {", 12 | " if (arg1 == arg2) {", 13 | " log.Print(\"They are the same\")", 14 | " }", 15 | " log.Printf(\"Checked equality\")", 16 | " while (arg1 < arg2) {", 17 | " arg1 += 1", 18 | " }", 19 | " log.Printf(\"Values are %d and %d\", arg1, arg2)", 20 | "}", 21 | } 22 | lines1 := []string{ 23 | "// Comments", 24 | "func my_method(arg_a int, arg_b int, arg_c string) {", 25 | " if (arg_a == arg_b) {", 26 | " log.Print(\"They are the same\")", 27 | " }", 28 | " while (arg_a < arg_b) { arg_a += 1 }", 29 | " log.Printf(\"Values are %d and %d\", arg_a, arg_b)", 30 | " log.Printf(\"Done!\")", 31 | "}", 32 | } 33 | lines2 := []string{""} 34 | 35 | // Regression test for OOB error in case of a really long patch 36 | lines3 := append(lines0[:1], append(make([]string, 100), lines0[1:]...)...) 37 | 38 | var cases = []struct { 39 | source_lines []string 40 | target_lines []string 41 | source_lineno int 42 | expectedOutput string 43 | }{ 44 | {lines0, lines1, 1, "2"}, 45 | {lines0, lines1, 2, "3"}, 46 | {lines0, lines1, 3, "4"}, 47 | {lines0, lines1, 4, "5"}, 48 | {lines0, lines1, 5, "5"}, // deleted line 49 | {lines0, lines1, 6, "6"}, // this and the next two lines have been collapsed 50 | {lines0, lines1, 7, "6"}, 51 | {lines0, lines1, 8, "6"}, 52 | {lines0, lines1, 9, "7"}, 53 | {lines0, lines1, 10, "9"}, 54 | {lines1, lines0, 1, "1"}, // deleted line 55 | {lines1, lines0, 2, "1"}, 56 | {lines1, lines0, 3, "2"}, 57 | {lines1, lines0, 4, "3"}, 58 | {lines1, lines0, 5, "4"}, 59 | {lines1, lines0, 6, "6"}, 60 | {lines1, lines0, 7, "9"}, 61 | {lines1, lines0, 8, "9"}, // deleted line 62 | {lines1, lines0, 9, "10"}, 63 | {lines0, lines2, 1, "1"}, // regression test 64 | {lines3, lines0, len(lines3), strconv.Itoa(len(lines0))}, // regression test 65 | } 66 | for _, testCase := range cases { 67 | target_lineno, err := analyzeEditAndMapLine(testCase.source_lines, testCase.target_lines, testCase.source_lineno) 68 | out := "" 69 | if err != nil { 70 | out = fmt.Sprint(err) 71 | } else { 72 | out = fmt.Sprint(target_lineno) 73 | } 74 | if out != testCase.expectedOutput { 75 | t.Error("Line", testCase.source_lineno, "failed", "\n Wanted", testCase.expectedOutput, "\n Got ", out) 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /server/json.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "encoding/json" 5 | ) 6 | 7 | type asJSON struct { 8 | v interface{} 9 | } 10 | 11 | func (j asJSON) String() string { 12 | b, e := json.Marshal(j.v) 13 | if e != nil { 14 | panic(e.Error()) 15 | } 16 | return string(b) 17 | } 18 | -------------------------------------------------------------------------------- /server/log/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["log.go"], 6 | importpath = "github.com/livegrep/livegrep/server/log", 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "//server/reqid:go_default_library", 10 | "@org_golang_x_net//context:go_default_library", 11 | ], 12 | ) 13 | -------------------------------------------------------------------------------- /server/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/livegrep/livegrep/server/reqid" 9 | "golang.org/x/net/context" 10 | ) 11 | 12 | func Printf(c context.Context, msg string, args ...interface{}) { 13 | var line bytes.Buffer 14 | line.WriteString(time.Now().UTC().Format("[2006-01-02T15:04:05.999] ")) 15 | 16 | if reqID, ok := reqid.FromContext(c); ok { 17 | fmt.Fprintf(&line, "[%s] ", reqID) 18 | } 19 | fmt.Fprintf(&line, msg, args...) 20 | fmt.Println(line.String()) 21 | } 22 | -------------------------------------------------------------------------------- /server/middleware/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["reverse_proxy.go"], 6 | importpath = "github.com/livegrep/livegrep/server/middleware", 7 | visibility = ["//visibility:public"], 8 | ) 9 | -------------------------------------------------------------------------------- /server/middleware/reverse_proxy.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | ) 7 | 8 | type reverseProxyHandler struct { 9 | inner http.Handler 10 | } 11 | 12 | func (h *reverseProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 13 | if ip := r.Header.Get("X-Forwarded-For"); len(ip) > 0 { 14 | r.RemoteAddr = strings.SplitN(ip, ",", 2)[0] 15 | } 16 | if proto := r.Header.Get("X-Forwarded-Proto"); proto == "http" { 17 | u := *r.URL 18 | u.Scheme = "https" 19 | u.Host = r.Host 20 | w.Header().Add("Location", u.String()) 21 | w.WriteHeader(http.StatusMovedPermanently) 22 | return 23 | } 24 | h.inner.ServeHTTP(w, r) 25 | } 26 | 27 | func UnwrapProxyHeaders(h http.Handler) http.Handler { 28 | return &reverseProxyHandler{h} 29 | } 30 | -------------------------------------------------------------------------------- /server/query.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "regexp" 8 | "strconv" 9 | "strings" 10 | "unicode/utf8" 11 | 12 | pb "github.com/livegrep/livegrep/src/proto/go_proto" 13 | ) 14 | 15 | var pieceRE = regexp.MustCompile(`\(|(?:^([a-zA-Z0-9-_]+):|\\.)| `) 16 | 17 | var knownTags = map[string]bool{ 18 | "file": true, 19 | "-file": true, 20 | "path": true, 21 | "-path": true, 22 | "repo": true, 23 | "-repo": true, 24 | "tags": true, 25 | "-tags": true, 26 | "case": true, 27 | "lit": true, 28 | "max_matches": true, 29 | } 30 | 31 | func onlyOneSynonym(ops map[string]string, op1 string, op2 string) (string, error) { 32 | if ops[op1] != "" && ops[op2] != "" { 33 | return "", fmt.Errorf("Cannot provide both %s: and %s:, because they are synonyms", op1, op2) 34 | } 35 | if ops[op1] != "" { 36 | return ops[op1], nil 37 | } 38 | return ops[op2], nil 39 | } 40 | 41 | func ParseQuery(query string, globalRegex bool) (pb.Query, error) { 42 | var out pb.Query 43 | 44 | ops := make(map[string]string) 45 | key := "" 46 | term := "" 47 | q := strings.TrimSpace(query) 48 | inRegex := globalRegex 49 | justGotSpace := true 50 | 51 | for { 52 | m := pieceRE.FindStringSubmatchIndex(q) 53 | if m == nil { 54 | term += q 55 | if _, alreadySet := ops[key]; alreadySet { 56 | return out, fmt.Errorf("got term twice: %s", key) 57 | } 58 | ops[key] = term 59 | break 60 | } 61 | 62 | term += q[:m[0]] 63 | match := q[m[0]:m[1]] 64 | q = q[m[1]:] 65 | 66 | justGotSpace = justGotSpace && m[0] == 0 67 | 68 | if match == " " { 69 | // A space: Ends the operator, if we're in one. 70 | if key == "" { 71 | term += " " 72 | 73 | } else { 74 | if _, alreadySet := ops[key]; alreadySet { 75 | return out, fmt.Errorf("got term twice: %s", key) 76 | } 77 | ops[key] = term 78 | key = "" 79 | term = "" 80 | inRegex = globalRegex 81 | } 82 | } else if match == "(" { 83 | if !(inRegex || justGotSpace) { 84 | term += "(" 85 | } else { 86 | // A parenthesis. Nothing is special until the 87 | // end of a balanced set of parenthesis 88 | p := 1 89 | i := 0 90 | esc := false 91 | var w bytes.Buffer 92 | for i < len(q) { 93 | // We decode runes ourselves instead 94 | // of using range because exiting the 95 | // loop with i = len(q) makes the edge 96 | // cases simpler. 97 | r, l := utf8.DecodeRuneInString(q[i:]) 98 | i += l 99 | switch { 100 | case esc: 101 | esc = false 102 | case r == '\\': 103 | esc = true 104 | case r == '(': 105 | p++ 106 | case r == ')': 107 | p-- 108 | } 109 | w.WriteRune(r) 110 | if p == 0 { 111 | break 112 | } 113 | } 114 | term += match + w.String() 115 | q = q[i:] 116 | } 117 | } else if match[0] == '\\' { 118 | term += match 119 | } else { 120 | // An operator. The key is in match group 1 121 | newKey := match[m[2]-m[0] : m[3]-m[0]] 122 | if key == "" && knownTags[newKey] { 123 | if strings.TrimSpace(term) != "" { 124 | if _, alreadySet := ops[key]; alreadySet { 125 | return out, fmt.Errorf("main search term must be contiguous") 126 | } 127 | ops[key] = term 128 | } 129 | term = "" 130 | key = newKey 131 | } else { 132 | term += match 133 | } 134 | if key == "lit" { 135 | inRegex = false 136 | } 137 | } 138 | justGotSpace = (match == " ") 139 | } 140 | 141 | var err error 142 | if out.File, err = onlyOneSynonym(ops, "file", "path"); err != nil { 143 | return out, err 144 | } 145 | out.Repo = ops["repo"] 146 | out.Tags = ops["tags"] 147 | if out.NotFile, err = onlyOneSynonym(ops, "-file", "-path"); err != nil { 148 | return out, err 149 | } 150 | out.NotRepo = ops["-repo"] 151 | out.NotTags = ops["-tags"] 152 | var bits []string 153 | for _, k := range []string{"", "case", "lit"} { 154 | bit := strings.TrimSpace(ops[k]) 155 | if k == "lit" || !globalRegex { 156 | bit = regexp.QuoteMeta(bit) 157 | } 158 | if len(bit) != 0 { 159 | bits = append(bits, bit) 160 | } 161 | } 162 | 163 | if len(bits) > 1 { 164 | return out, errors.New("You cannot provide multiple of case:, lit:, and a bare regex") 165 | } 166 | 167 | if len(bits) > 0 { 168 | out.Line = bits[0] 169 | } 170 | 171 | if !globalRegex { 172 | out.File = regexp.QuoteMeta(out.File) 173 | out.NotFile = regexp.QuoteMeta(out.NotFile) 174 | out.Repo = regexp.QuoteMeta(out.Repo) 175 | out.NotRepo = regexp.QuoteMeta(out.NotRepo) 176 | } 177 | 178 | if out.Line == "" && out.File != "" { 179 | out.Line = out.File 180 | out.File = "" 181 | out.FilenameOnly = true 182 | } 183 | 184 | if _, ok := ops["case"]; ok { 185 | out.FoldCase = false 186 | } else if _, ok := ops["lit"]; ok { 187 | out.FoldCase = false 188 | } else { 189 | out.FoldCase = strings.IndexAny(out.Line, "ABCDEFGHIJKLMNOPQRSTUVWXYZ") == -1 190 | } 191 | if v, ok := ops["max_matches"]; ok && v != "" { 192 | i, err := strconv.Atoi(v) 193 | if err == nil { 194 | out.MaxMatches = int32(i) 195 | } else { 196 | return out, errors.New("Value given to max_matches: must be a valid integer") 197 | } 198 | } else { 199 | out.MaxMatches = 0 200 | } 201 | 202 | return out, nil 203 | } 204 | -------------------------------------------------------------------------------- /server/query_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | pb "github.com/livegrep/livegrep/src/proto/go_proto" 8 | ) 9 | 10 | func TestParseQuery(t *testing.T) { 11 | cases := []struct { 12 | in string 13 | out pb.Query 14 | regex bool 15 | }{ 16 | // regex parse mode 17 | { 18 | "hello", 19 | pb.Query{Line: "hello", FoldCase: true}, 20 | true, 21 | }, 22 | { 23 | "a b c", 24 | pb.Query{Line: "a b c", FoldCase: true}, 25 | true, 26 | }, 27 | { 28 | "line file:.rb", 29 | pb.Query{ 30 | Line: "line", 31 | File: ".rb", 32 | FoldCase: true, 33 | }, 34 | true, 35 | }, 36 | { 37 | " a ", 38 | pb.Query{Line: "a", FoldCase: true}, 39 | true, 40 | }, 41 | { 42 | "( a )", 43 | pb.Query{Line: "( a )", FoldCase: true}, 44 | true, 45 | }, 46 | { 47 | "Aa", 48 | pb.Query{Line: "Aa", FoldCase: false}, 49 | true, 50 | }, 51 | { 52 | "case:abc", 53 | pb.Query{Line: "abc", FoldCase: false}, 54 | true, 55 | }, 56 | { 57 | "case:abc file:^kernel/", 58 | pb.Query{Line: "abc", FoldCase: false, File: "^kernel/"}, 59 | true, 60 | }, 61 | { 62 | "case:abc file:( )", 63 | pb.Query{Line: "abc", FoldCase: false, File: "( )"}, 64 | true, 65 | }, 66 | { 67 | "( () ( ", 68 | pb.Query{Line: "( () (", FoldCase: true}, 69 | true, 70 | }, 71 | { 72 | `a file:\(`, 73 | pb.Query{Line: "a", File: `\(`, FoldCase: true}, 74 | true, 75 | }, 76 | { 77 | `a file:(\()`, 78 | pb.Query{Line: "a", File: `(\()`, FoldCase: true}, 79 | true, 80 | }, 81 | { 82 | `(`, 83 | pb.Query{Line: "(", FoldCase: true}, 84 | true, 85 | }, 86 | { 87 | `(file:)`, 88 | pb.Query{Line: "(file:)", FoldCase: true}, 89 | true, 90 | }, 91 | { 92 | `re tags:kind:function`, 93 | pb.Query{Line: "re", FoldCase: true, Tags: "kind:function"}, 94 | true, 95 | }, 96 | { 97 | `-file:Godep re`, 98 | pb.Query{Line: "re", NotFile: "Godep", FoldCase: true}, 99 | true, 100 | }, 101 | { 102 | `-file:. -repo:Godep re`, 103 | pb.Query{Line: "re", NotFile: ".", NotRepo: "Godep", FoldCase: true}, 104 | true, 105 | }, 106 | { 107 | `-tags:kind:class re`, 108 | pb.Query{Line: "re", NotTags: "kind:class", FoldCase: true}, 109 | true, 110 | }, 111 | { 112 | `case:foo:`, 113 | pb.Query{Line: "foo:", FoldCase: false}, 114 | true, 115 | }, 116 | { 117 | `lit:.`, 118 | pb.Query{Line: `\.`, FoldCase: false}, 119 | true, 120 | }, 121 | { 122 | `std::string`, 123 | pb.Query{Line: `std::string`, FoldCase: true}, 124 | true, 125 | }, 126 | { 127 | `a max_matches:100`, 128 | pb.Query{Line: "a", FoldCase: true, MaxMatches: 100}, 129 | true, 130 | }, 131 | { 132 | `a max_matches:`, 133 | pb.Query{Line: "a", FoldCase: true}, 134 | true, 135 | }, 136 | { 137 | `file:hello`, 138 | pb.Query{Line: "hello", FoldCase: true, FilenameOnly: true}, 139 | true, 140 | }, 141 | { 142 | `file:HELLO`, 143 | pb.Query{Line: "HELLO", FoldCase: false, FilenameOnly: true}, 144 | true, 145 | }, 146 | { 147 | `lit:a( file:b`, 148 | pb.Query{Line: `a\(`, File: "b", FoldCase: false}, 149 | true, 150 | }, 151 | { 152 | `lit:a(b file:c`, 153 | pb.Query{Line: `a\(b`, File: "c", FoldCase: false}, 154 | true, 155 | }, 156 | 157 | // literal parse mode 158 | { 159 | "a( file:b", 160 | pb.Query{Line: `a\(`, File: "b", FoldCase: true}, 161 | false, 162 | }, 163 | { 164 | "a (file:b", 165 | pb.Query{Line: `a \(file:b`, FoldCase: true}, 166 | false, 167 | }, 168 | { 169 | "(file:a b", 170 | pb.Query{Line: `\(file:a b`, FoldCase: true}, 171 | false, 172 | }, 173 | { 174 | "(file:a) b", 175 | pb.Query{Line: `\(file:a\) b`, FoldCase: true}, 176 | false, 177 | }, 178 | { 179 | "(file:a repo:b", 180 | pb.Query{Line: `\(file:a repo:b`, FoldCase: true}, 181 | false, 182 | }, 183 | { 184 | "(file:a) repo:b", 185 | pb.Query{Line: `\(file:a\)`, Repo: "b", FoldCase: true}, 186 | false, 187 | }, 188 | { 189 | "(file:a) (repo:b)", 190 | pb.Query{Line: `\(file:a\) \(repo:b\)`, FoldCase: true}, 191 | false, 192 | }, 193 | { 194 | "file:a( b", 195 | pb.Query{Line: `b`, File: `a\(`, FoldCase: true}, 196 | false, 197 | }, 198 | } 199 | 200 | for _, tc := range cases { 201 | parsed, err := ParseQuery(tc.in, tc.regex) 202 | if !reflect.DeepEqual(tc.out, parsed) { 203 | t.Errorf("error parsing %q: expected %#v got %#v", 204 | tc.in, tc.out, parsed) 205 | } 206 | if err != nil { 207 | t.Errorf("parse(%v) error=%v", tc.in, err) 208 | } 209 | } 210 | } 211 | 212 | func TestParseQueryError(t *testing.T) { 213 | cases := []struct { 214 | in string 215 | }{ 216 | {"case:a b"}, 217 | {"lit:a b"}, 218 | {"case:a lit:b"}, 219 | {"a max_matches:a"}, 220 | {"a file:b c"}, 221 | {"a file:((abc()())()) c"}, 222 | } 223 | 224 | for _, tc := range cases { 225 | parsed, err := ParseQuery(tc.in, true) 226 | if err == nil { 227 | t.Errorf("expected an error parsing (%v), got %#v", tc.in, parsed) 228 | } 229 | } 230 | } 231 | -------------------------------------------------------------------------------- /server/reqid/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["reqid.go"], 6 | importpath = "github.com/livegrep/livegrep/server/reqid", 7 | visibility = ["//visibility:public"], 8 | deps = ["@org_golang_x_net//context:go_default_library"], 9 | ) 10 | -------------------------------------------------------------------------------- /server/reqid/reqid.go: -------------------------------------------------------------------------------- 1 | package reqid 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/hex" 6 | "fmt" 7 | 8 | "golang.org/x/net/context" 9 | ) 10 | 11 | type key int 12 | 13 | const reqIDKey key = 0 14 | 15 | type RequestID string 16 | 17 | func New() RequestID { 18 | bytes := make([]byte, 16) 19 | if _, err := rand.Read(bytes); err != nil { 20 | panic(fmt.Sprintf("rand.Read: %v", err)) 21 | } 22 | return RequestID(hex.EncodeToString(bytes)) 23 | } 24 | 25 | func NewContext(ctx context.Context, reqID RequestID) context.Context { 26 | return context.WithValue(ctx, reqIDKey, reqID) 27 | } 28 | 29 | func FromContext(ctx context.Context) (RequestID, bool) { 30 | reqID, ok := ctx.Value(reqIDKey).(RequestID) 31 | return reqID, ok 32 | } 33 | -------------------------------------------------------------------------------- /server/server_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "regexp" 5 | "testing" 6 | ) 7 | 8 | func assertRepoPath(t *testing.T, 9 | repoRegex *regexp.Regexp, 10 | url string, 11 | expectedRepo string, 12 | expectedPath string, 13 | expectedErr error) { 14 | actualRepo, actualPath, err := getRepoPathFromURL(repoRegex, url) 15 | if err != expectedErr { 16 | t.Errorf("error expectation mismatch when parsing url, got %v, expected %v", err.Error(), expectedErr) 17 | } 18 | 19 | if actualRepo != expectedRepo { 20 | t.Errorf("repo expectation mismatch when parsing url, got %v, expected %v", actualRepo, expectedRepo) 21 | } 22 | 23 | if actualPath != expectedPath { 24 | t.Errorf("repo expectation mismatch when parsing url, got %v, expected %v", actualPath, expectedPath) 25 | } 26 | } 27 | 28 | func TestRepoRegexParsing(t *testing.T) { 29 | repoNames := []string{"test-repo", "test-org/test-repo", "test-repo-2", "foobar"} 30 | 31 | repoRegex, err := buildRepoRegex(repoNames) 32 | if err != nil { 33 | t.Errorf("unexpected error building repo regex (%v)", err.Error()) 34 | } 35 | 36 | assertRepoPath(t, repoRegex, "/view/test-repo/path/to/foobar.css", "test-repo", "path/to/foobar.css", nil) 37 | assertRepoPath(t, repoRegex, "/view/test-org/test-repo/path/to/foobar.css", "test-org/test-repo", "path/to/foobar.css", nil) 38 | assertRepoPath(t, repoRegex, "/view/test-repo-2/path/to/foobar.css", "test-repo-2", "path/to/foobar.css", nil) 39 | assertRepoPath(t, repoRegex, "/view/foobar/path/to/foobar.css", "foobar", "path/to/foobar.css", nil) 40 | assertRepoPath(t, repoRegex, "/view/not-exist/path/to/foobar.css", "", "", serveUrlParseError) 41 | assertRepoPath(t, repoRegex, "/not/even/a/url/not-exist/path/to/foobar.css", "", "", serveUrlParseError) 42 | } 43 | -------------------------------------------------------------------------------- /server/templates/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["templates.go"], 6 | importpath = "github.com/livegrep/livegrep/server/templates", 7 | visibility = ["//visibility:public"], 8 | deps = ["//blameworthy:go_default_library"], 9 | ) 10 | -------------------------------------------------------------------------------- /server/templates/templates.go: -------------------------------------------------------------------------------- 1 | package templates 2 | 3 | import ( 4 | "bufio" 5 | "encoding/base64" 6 | "encoding/hex" 7 | "fmt" 8 | "html/template" 9 | "net/url" 10 | "os" 11 | "path/filepath" 12 | "regexp" 13 | "strings" 14 | 15 | "github.com/livegrep/livegrep/blameworthy" 16 | ) 17 | 18 | var possibleURL = regexp.MustCompile( 19 | `\bhttps?://[A-Za-z0-9\-._~:/?#\[\]@!$&'()*+,;=]+`, 20 | ) 21 | 22 | func prettyCommit(c *blameworthy.Commit) string { 23 | if len(c.Author) > 0 && c.Date > 0 { 24 | return fmt.Sprintf("%04d-%02d-%02d %.8s", 25 | c.Date/10000, c.Date%10000/100, c.Date%100, 26 | c.Author) 27 | } 28 | return c.Hash + " " // turn 16 characters into 19 29 | } 30 | 31 | func TurnURLsIntoLinks(s string) template.HTML { 32 | // Instead of using a complex RE that matches only valid URLs, 33 | // let's match anything vaguely URL-like, then use Go's URL 34 | // parser to decide whether it's a URL. 35 | matches := possibleURL.FindAllStringIndex(s, -1) 36 | i := 0 37 | h := []string{} 38 | for _, match := range matches { 39 | j := match[0] 40 | k := match[1] 41 | h = append(h, template.HTMLEscapeString(s[i:j])) 42 | u := s[j:k] 43 | _, err := url.Parse(u) 44 | if err != nil { 45 | h = append(h, template.HTMLEscapeString(u)) 46 | } else { 47 | h = append(h, "") 52 | h = append(h, template.HTMLEscapeString(u)) 53 | h = append(h, "") 54 | } 55 | i = k 56 | } 57 | h = append(h, template.HTMLEscapeString(s[i:len(s)])) 58 | return template.HTML(strings.Join(h, "")) 59 | } 60 | 61 | func linkTag(nonce template.HTMLAttr, rel string, s string, m map[string]string) template.HTML { 62 | hash := m[strings.TrimPrefix(s, "/")] 63 | href := s + "?v=" + hash 64 | hashBytes, _ := hex.DecodeString(hash) 65 | integrity := "sha256-" + base64.StdEncoding.EncodeToString(hashBytes) 66 | return template.HTML(fmt.Sprintf( 67 | ``, 68 | nonce, rel, href, integrity, 69 | )) 70 | } 71 | 72 | func scriptTag(nonce template.HTMLAttr, s string, m map[string]string) template.HTML { 73 | hash := m[strings.TrimPrefix(s, "/")] 74 | href := s + "?v=" + hash 75 | hashBytes, _ := hex.DecodeString(hash) 76 | integrity := "sha256-" + base64.StdEncoding.EncodeToString(hashBytes) 77 | return template.HTML(fmt.Sprintf( 78 | ``, 79 | nonce, href, integrity, 80 | )) 81 | } 82 | 83 | func getFuncs() map[string]interface{} { 84 | return map[string]interface{}{ 85 | "loop": func(n int) []struct{} { return make([]struct{}, n) }, 86 | "toLineNum": func(n int) int { return n + 1 }, 87 | "prettyCommit": prettyCommit, 88 | "linkTag": linkTag, 89 | "scriptTag": scriptTag, 90 | } 91 | } 92 | 93 | func LoadTemplates(base string, templates map[string]*template.Template) error { 94 | pattern := base + "/templates/common/*.html" 95 | common := template.New("").Funcs(getFuncs()) 96 | common = template.Must(common.ParseGlob(pattern)) 97 | 98 | pattern = base + "/templates/*.html" 99 | paths, err := filepath.Glob(pattern) 100 | if err != nil { 101 | return err 102 | } 103 | for _, path := range paths { 104 | t := template.Must(common.Clone()) 105 | t = template.Must(t.ParseFiles(path)) 106 | templates[filepath.Base(path)] = t 107 | } 108 | return nil 109 | } 110 | 111 | func LoadAssetHashes(assetHashFile string, assetHashMap map[string]string) error { 112 | file, err := os.Open(assetHashFile) 113 | if err != nil { 114 | return err 115 | } 116 | defer file.Close() 117 | 118 | scanner := bufio.NewScanner(file) 119 | 120 | for k := range assetHashMap { 121 | delete(assetHashMap, k) 122 | } 123 | 124 | for scanner.Scan() { 125 | pieces := strings.SplitN(scanner.Text(), " ", 2) 126 | hash := pieces[0] 127 | asset := pieces[1] 128 | (assetHashMap)[asset] = hash 129 | } 130 | 131 | return nil 132 | } 133 | -------------------------------------------------------------------------------- /src/BUILD: -------------------------------------------------------------------------------- 1 | cc_library( 2 | name = "codesearch", 3 | srcs = glob([ 4 | "*.cc", 5 | ]), 6 | hdrs = glob(["*.h"]), 7 | copts = [ 8 | "-Wno-sign-compare", 9 | "-std=c++14", 10 | ], 11 | visibility = ["//visibility:public"], 12 | deps = [ 13 | "//src/lib", 14 | "//src/proto:cc_config_proto", 15 | "//third_party:utf8cpp", 16 | "@boost//:filesystem", 17 | "@boost//:intrusive_ptr", 18 | "@com_github_libgit2//:libgit2", 19 | "@com_google_absl//absl/container:flat_hash_set", 20 | "@com_google_absl//absl/hash", 21 | "@com_google_absl//absl/strings", 22 | "@com_googlesource_code_re2//:re2", 23 | "@divsufsort", 24 | ], 25 | ) 26 | -------------------------------------------------------------------------------- /src/chunk.cc: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- chunk.cc 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #include "src/lib/radix_sort.h" 9 | #include "src/lib/metrics.h" 10 | 11 | #include "src/chunk.h" 12 | #include "src/codesearch.h" 13 | 14 | #include "divsufsort.h" 15 | #include "re2/re2.h" 16 | #include 17 | 18 | #include 19 | 20 | using re2::StringPiece; 21 | 22 | DECLARE_bool(index); 23 | 24 | void chunk::add_chunk_file(indexed_file *sf, const StringPiece& line) 25 | { 26 | int l = (unsigned char*)line.data() - data; 27 | int r = l + line.size(); 28 | chunk_file *f = NULL; 29 | int min_dist = numeric_limits::max(), dist; 30 | for (vector::iterator it = cur_file.begin(); 31 | it != cur_file.end(); it ++) { 32 | if (l <= it->left) 33 | dist = max(0, it->left - r); 34 | else if (r >= it->right) 35 | dist = max(0, l - it->right); 36 | else 37 | dist = 0; 38 | assert(dist == 0 || r < it->left || l > it->right); 39 | if (dist < min_dist) { 40 | min_dist = dist; 41 | f = &(*it); 42 | } 43 | } 44 | if (f && min_dist < kMaxGap) { 45 | f->expand(l, r); 46 | return; 47 | } 48 | chunk_files++; 49 | cur_file.push_back(chunk_file()); 50 | chunk_file& cf = cur_file.back(); 51 | cf.files.push_front(sf); 52 | cf.left = l; 53 | cf.right = r; 54 | } 55 | 56 | void chunk::finish_file() { 57 | int right = -1; 58 | sort(cur_file.begin(), cur_file.end()); 59 | for (vector::iterator it = cur_file.begin(); 60 | it != cur_file.end(); it ++) { 61 | assert(right < it->left); 62 | right = max(right, it->right); 63 | } 64 | files.insert(files.end(), cur_file.begin(), cur_file.end()); 65 | cur_file.clear(); 66 | } 67 | 68 | int chunk::chunk_files = 0; 69 | 70 | void chunk::finalize() { 71 | if (FLAGS_index) { 72 | // For the purposes of livegrep's line-based sorting, we need 73 | // to sort \n before all other characters. divsufsort, 74 | // understandably, just lexically-sorts sorts thing. Kludge 75 | // around by munging the data in-place before and after the 76 | // sort. Sorting must look at all the data anyways, so this is 77 | // not an overly-expensive job. 78 | std::replace(data, data + size, '\n', '\0'); 79 | divsufsort(data, reinterpret_cast(suffixes), size); 80 | std::replace(data, data + size, '\0', '\n'); 81 | } 82 | } 83 | 84 | void chunk::finalize_files() { 85 | sort(files.begin(), files.end()); 86 | 87 | vector::iterator out, in; 88 | out = in = files.begin(); 89 | while (in != files.end()) { 90 | *out = *in; 91 | ++in; 92 | while (in != files.end() && 93 | out->left == in->left && 94 | out->right == in->right) { 95 | out->files.push_back(in->files.front()); 96 | ++in; 97 | } 98 | ++out; 99 | } 100 | files.resize(out - files.begin()); 101 | 102 | build_tree_names(); 103 | build_tree(); 104 | } 105 | 106 | void chunk::build_tree_names() { 107 | for (auto it = files.begin(); it != files.end(); it++) { 108 | for (auto it2 = it->files.begin(); it2 != it->files.end(); it2++) { 109 | tree_names.insert((*it2)->tree->name); 110 | } 111 | } 112 | } 113 | 114 | void chunk::build_tree() { 115 | assert(is_sorted(files.begin(), files.end())); 116 | cf_root = build_tree(0, files.size()); 117 | } 118 | 119 | unique_ptr chunk::build_tree(int left, int right) { 120 | if (right == left) 121 | return 0; 122 | int mid = (left + right) / 2; 123 | auto node = std::make_unique(); 124 | 125 | node->chunk = &files[mid]; 126 | node->left = build_tree(left, mid); 127 | node->right = build_tree(mid + 1, right); 128 | node->right_limit = node->chunk->right; 129 | if (node->left && node->left->right_limit > node->right_limit) 130 | node->right_limit = node->left->right_limit; 131 | if (node->right && node->right->right_limit > node->right_limit) 132 | node->right_limit = node->right->right_limit; 133 | assert(!node->left || *(node->left->chunk) < *(node->chunk)); 134 | assert(!node->right || *(node->chunk) < *(node->right->chunk)); 135 | return node; 136 | } 137 | -------------------------------------------------------------------------------- /src/chunk.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- chunk.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_CHUNK_H 9 | #define CODESEARCH_CHUNK_H 10 | 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | 24 | struct indexed_file; 25 | namespace re2 { 26 | class StringPiece; 27 | } 28 | 29 | using namespace std; 30 | using re2::StringPiece; 31 | 32 | /* 33 | * A chunk_file in a given chunk's `files' list means that some or all 34 | * of bytes `left' through `right' (inclusive on both sides) in 35 | * chunk->data are present in each of chunk->files. 36 | */ 37 | struct chunk_file { 38 | list files; 39 | int left; 40 | int right; 41 | void expand(int l, int r) { 42 | left = min(left, l); 43 | right = max(right, r); 44 | } 45 | 46 | bool operator<(const chunk_file& rhs) const { 47 | return left < rhs.left || (left == rhs.left && right < rhs.right); 48 | } 49 | }; 50 | 51 | const size_t kMaxGap = 1 << 10; 52 | 53 | struct chunk_file_node { 54 | chunk_file *chunk; 55 | int right_limit; 56 | 57 | std::unique_ptr left, right; 58 | }; 59 | 60 | struct chunk { 61 | // total number of chunk_file objects across all chunks. 62 | static int chunk_files; 63 | 64 | int id; // Sequential id 65 | int size; 66 | 67 | // Collects references to all files which contain lines stored in this 68 | // chunk's data. Sorted (and compacted) at the very end of index creation. 69 | vector files; 70 | 71 | // Collects the names of all trees indexed in this chunk, to enable 72 | // short-circuiting based on a repo constraint. 73 | set tree_names; 74 | 75 | // Transient during index creation. Collects references to the file 76 | // currently being processed by the code_searcher, when that file contains 77 | // lines stored in this chunk's data. One the code_searcher finishes 78 | // processing each file, any references here are merged into `files` by 79 | // finish_file(), and this vector is cleared. 80 | vector cur_file; 81 | 82 | // BST constructed from `files` at the very end of index creation. Used to 83 | // efficiently find, given a substring of this chunk's data, the files 84 | // might contain that substring. 85 | unique_ptr cf_root; 86 | 87 | // The suffix array; constructed from `data` during finalization (once the 88 | // chunk's data block is full, but before all files have been processed). 89 | uint32_t *suffixes; 90 | 91 | // Many lines of code, from many files, concatenated together. 92 | unsigned char *data; 93 | 94 | chunk(unsigned char *data, uint32_t *suffixes) 95 | : size(0), files(), cf_root(), 96 | suffixes(suffixes), data(data) { } 97 | 98 | void add_chunk_file(indexed_file *sf, const StringPiece& line); 99 | void finish_file(); 100 | void finalize(); 101 | void finalize_files(); 102 | void build_tree_names(); 103 | void build_tree(); 104 | 105 | struct lt_suffix { 106 | const chunk *chunk_; 107 | lt_suffix(const chunk *chunk) : chunk_(chunk) { } 108 | bool operator()(uint32_t lhs, uint32_t rhs) { 109 | const unsigned char *l = &chunk_->data[lhs]; 110 | const unsigned char *r = &chunk_->data[rhs]; 111 | const unsigned char *le = static_cast 112 | (memchr(l, '\n', chunk_->size - lhs)); 113 | const unsigned char *re = static_cast 114 | (memchr(r, '\n', chunk_->size - rhs)); 115 | assert(le); 116 | assert(re); 117 | return memcmp(l, r, min(le - l, re - r)) < 0; 118 | } 119 | 120 | bool operator()(uint32_t lhs, const string& rhs) { 121 | return cmp(lhs, rhs) < 0; 122 | } 123 | 124 | bool operator()(const string& lhs, uint32_t rhs) { 125 | return cmp(rhs, lhs) > 0; 126 | } 127 | 128 | private: 129 | int cmp(uint32_t lhs, const string& rhs) { 130 | const unsigned char *l = &chunk_->data[lhs]; 131 | const unsigned char *le = static_cast 132 | (memchr(l, '\n', chunk_->size - lhs)); 133 | size_t lhs_len = le - l; 134 | int cmp = memcmp(l, rhs.c_str(), min(lhs_len, rhs.size())); 135 | if (cmp == 0) 136 | return lhs_len < rhs.size() ? -1 : 0; 137 | return cmp; 138 | } 139 | }; 140 | 141 | unique_ptr build_tree(int left, int right); 142 | 143 | private: 144 | chunk(const chunk&); 145 | chunk operator=(const chunk&); 146 | }; 147 | 148 | extern size_t kChunkSpace; 149 | 150 | #endif 151 | -------------------------------------------------------------------------------- /src/chunk_allocator.cc: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- chunk_allocator.cc 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #include "src/lib/debug.h" 9 | 10 | #include "src/chunk_allocator.h" 11 | #include "src/chunk.h" 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | DECLARE_int32(threads); 19 | DECLARE_bool(index); 20 | DEFINE_int32(chunk_power, 27, "Size of search chunks, as a power of two"); 21 | size_t kChunkSize = (1 << 27); 22 | 23 | static bool validate_chunk_power(const char* flagname, int32_t value) { 24 | if (value > 10 && value < 30) { 25 | kChunkSize = (1 << value); 26 | return true; 27 | } 28 | return false; 29 | } 30 | 31 | static const bool dummy = gflags::RegisterFlagValidator(&FLAGS_chunk_power, 32 | validate_chunk_power); 33 | 34 | void chunk_allocator::finalize_worker(chunk_allocator *alloc) { 35 | chunk *c; 36 | while (alloc->finalize_queue_.pop(&c)) { 37 | c->finalize(); 38 | } 39 | } 40 | 41 | chunk_allocator::chunk_allocator() : 42 | chunk_size_(kChunkSize), content_finger_(0), current_(0) { 43 | for (int i = 0; i < FLAGS_threads; ++i) 44 | threads_.emplace_back(finalize_worker, this); 45 | } 46 | 47 | chunk_allocator::~chunk_allocator() { 48 | finalize_queue_.close(); 49 | for (auto it = threads_.begin(); it != threads_.end(); ++it) 50 | it->join(); 51 | } 52 | 53 | void chunk_allocator::set_chunk_size(size_t size) { 54 | assert(current_ == 0); 55 | assert(!chunks_.size()); 56 | chunk_size_ = size; 57 | } 58 | 59 | void chunk_allocator::cleanup() { 60 | for (auto c = begin(); c != end(); ++ c) 61 | free_chunk(*c); 62 | } 63 | 64 | unsigned char *chunk_allocator::alloc(size_t len) { 65 | assert(len < chunk_size_); 66 | if (current_ == 0 || (current_->size + len) > chunk_size_) 67 | new_chunk(); 68 | unsigned char *out = current_->data + current_->size; 69 | current_->size += len; 70 | return out; 71 | } 72 | 73 | uint8_t *chunk_allocator::alloc_content_data(size_t len) { 74 | if (len >= kContentChunkSize) 75 | return 0; 76 | if (content_finger_ == 0 || (content_finger_ + len > content_chunks_.back().end)) { 77 | if (content_finger_) 78 | content_chunks_.back().end = content_finger_; 79 | content_chunks_.push_back(alloc_content_chunk()); 80 | content_finger_ = content_chunks_.back().data; 81 | } 82 | uint8_t *out = content_finger_; 83 | content_finger_ += len; 84 | assert(content_finger_ > content_chunks_.back().data && 85 | content_finger_ <= content_chunks_.back().end); 86 | return out; 87 | } 88 | 89 | void chunk_allocator::finish_chunk() { 90 | if (current_) { 91 | finalize_queue_.push(current_); 92 | } 93 | } 94 | 95 | void chunk_allocator::new_chunk() { 96 | finish_chunk(); 97 | current_ = alloc_chunk(); 98 | madvise(current_->data, chunk_size_, MADV_RANDOM); 99 | madvise(current_->suffixes, chunk_size_ * sizeof(*current_->suffixes), MADV_RANDOM); 100 | current_->id = chunks_.size(); 101 | by_data_[current_->data] = current_; 102 | chunks_.push_back(current_); 103 | } 104 | 105 | void chunk_allocator::finalize() { 106 | if (!current_) 107 | return; 108 | finish_chunk(); 109 | finalize_queue_.close(); 110 | for (auto it = threads_.begin(); it != threads_.end(); ++it) 111 | it->join(); 112 | threads_.clear(); 113 | for (auto it = begin(); it != end(); ++it) 114 | (*it)->finalize_files(); 115 | if (content_finger_) 116 | content_chunks_.back().end = content_finger_; 117 | } 118 | 119 | void chunk_allocator::skip_chunk() { 120 | current_ = 0; 121 | new_chunk(); 122 | } 123 | 124 | void chunk_allocator::drop_caches() { 125 | } 126 | 127 | chunk *chunk_allocator::chunk_from_string(const unsigned char *p) { 128 | auto it = by_data_.lower_bound(p); 129 | if (it == by_data_.end() || it->first != p) { 130 | assert(it != by_data_.begin()); 131 | --it; 132 | } 133 | assert(it->first <= p && p <= it->first + it->second->size); 134 | return it->second; 135 | } 136 | 137 | class mem_allocator : public chunk_allocator { 138 | public: 139 | virtual chunk *alloc_chunk() { 140 | unsigned char *buf = new unsigned char[chunk_size_]; 141 | uint32_t *idx = FLAGS_index ? new uint32_t[chunk_size_] : 0; 142 | return new chunk(buf, idx); 143 | } 144 | 145 | virtual buffer alloc_content_chunk() { 146 | uint8_t *buf = new uint8_t[kContentChunkSize]; 147 | return (buffer){ buf, buf + kContentChunkSize }; 148 | } 149 | 150 | virtual void free_chunk(chunk *chunk) { 151 | delete[] chunk->data; 152 | delete[] chunk->suffixes; 153 | delete chunk; 154 | } 155 | }; 156 | 157 | unique_ptr make_mem_allocator() { 158 | return make_unique(); 159 | } 160 | -------------------------------------------------------------------------------- /src/chunk_allocator.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- chunk_allocator.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_CHUNK_ALLOCATOR_H 9 | #define CODESEARCH_CHUNK_ALLOCATOR_H 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "src/lib/thread_queue.h" 18 | 19 | using namespace std; 20 | struct chunk; 21 | class code_searcher; 22 | 23 | struct buffer { 24 | uint8_t *data; 25 | uint8_t *end; 26 | }; 27 | 28 | class chunk_allocator { 29 | public: 30 | chunk_allocator(); 31 | virtual ~chunk_allocator(); 32 | void cleanup(); 33 | 34 | void set_chunk_size(size_t size); 35 | size_t chunk_size() { 36 | return chunk_size_; 37 | } 38 | 39 | unsigned char *alloc(size_t len); 40 | uint8_t *alloc_content_data(size_t len); 41 | 42 | vector::iterator begin () { 43 | return chunks_.begin(); 44 | } 45 | 46 | vector::iterator end () { 47 | return chunks_.end(); 48 | } 49 | 50 | vector::const_iterator begin_content() { 51 | return content_chunks_.begin(); 52 | } 53 | 54 | vector::const_iterator end_content() { 55 | return content_chunks_.end(); 56 | } 57 | 58 | chunk *at(size_t i) { 59 | assert(i < chunks_.size()); 60 | return chunks_[i]; 61 | } 62 | 63 | size_t size () { 64 | return chunks_.size(); 65 | } 66 | 67 | chunk *current_chunk() { 68 | return current_; 69 | } 70 | 71 | void skip_chunk(); 72 | virtual void finalize(); 73 | 74 | chunk *chunk_from_string(const unsigned char *p); 75 | 76 | virtual void drop_caches(); 77 | protected: 78 | static void finalize_worker(chunk_allocator *); 79 | 80 | virtual chunk *alloc_chunk() = 0; 81 | virtual void free_chunk(chunk *chunk) = 0; 82 | virtual buffer alloc_content_chunk() = 0; 83 | void finish_chunk(); 84 | void new_chunk(); 85 | 86 | size_t chunk_size_; 87 | vector chunks_; 88 | vector content_chunks_; 89 | 90 | // Subsequent fields are transient (only used during index creation). 91 | 92 | // Tracks how much of the current content chunk has been allocated by 93 | // alloc_content_data(). 94 | uint8_t *content_finger_; 95 | 96 | // Points to the chunk currently being filled (which is also chunks_.back()). 97 | chunk *current_; 98 | 99 | // Machinery to finalize chunks (i.e. build the suffix array from the data) 100 | // in the background. 101 | thread_queue finalize_queue_; 102 | vector threads_; 103 | 104 | // Used by chunk_from_string() to efficiently find the chunk containing an 105 | // already-indexed line of code. 106 | map by_data_; 107 | }; 108 | 109 | const size_t kContentChunkSize = (1UL << 22); 110 | 111 | #endif 112 | -------------------------------------------------------------------------------- /src/common.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- common.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_COMMON_H 9 | #define CODESEARCH_COMMON_H 10 | 11 | typedef unsigned char uchar; 12 | 13 | #endif 14 | -------------------------------------------------------------------------------- /src/content.cc: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- content.cc 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #include "src/content.h" 9 | #include "src/chunk.h" 10 | 11 | void file_contents_builder::extend(chunk *c, const StringPiece &piece) { 12 | if (pieces_.size() && piece.size()) { 13 | if (pieces_.back().data() + pieces_.back().size() == piece.data()) { 14 | pieces_.back().set(pieces_.back().data(), 15 | piece.size() + pieces_.back().size()); 16 | return; 17 | } 18 | } 19 | 20 | pieces_.push_back(piece); 21 | } 22 | 23 | file_contents *file_contents_builder::build(chunk_allocator *alloc) { 24 | size_t len = sizeof(uint32_t) * (1 + 3*pieces_.size()); 25 | file_contents *out = new(alloc->alloc_content_data(len)) file_contents(pieces_.size()); 26 | if (out == 0) 27 | return 0; 28 | for (int i = 0; i < pieces_.size(); i++) { 29 | const unsigned char *p = reinterpret_cast 30 | (pieces_[i].data()); 31 | chunk *chunk = alloc->chunk_from_string(p); 32 | out->pieces_[i].chunk = chunk->id; 33 | out->pieces_[i].off = p - chunk->data; 34 | out->pieces_[i].len = pieces_[i].size(); 35 | } 36 | return out; 37 | } 38 | -------------------------------------------------------------------------------- /src/content.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- content.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_CONTENT_H 9 | #define CODESEARCH_CONTENT_H 10 | 11 | #include 12 | #include "re2/re2.h" 13 | 14 | #include "src/chunk.h" 15 | #include "src/chunk_allocator.h" 16 | 17 | using re2::StringPiece; 18 | using std::vector; 19 | 20 | 21 | class file_contents { 22 | public: 23 | struct piece { 24 | uint32_t chunk; 25 | uint32_t off; 26 | uint32_t len; 27 | } __attribute__((packed)); 28 | 29 | template 30 | class proxy { 31 | T obj; 32 | public: 33 | proxy(T obj) : obj(obj) {} 34 | T *operator->() { 35 | return &obj; 36 | } 37 | }; 38 | class iterator { 39 | public: 40 | const StringPiece operator*() { 41 | return StringPiece(reinterpret_cast(alloc_->at(it_->chunk)->data + it_->off), 42 | it_->len); 43 | } 44 | 45 | proxy operator->() { 46 | return proxy(this->operator*()); 47 | } 48 | 49 | iterator &operator++() { 50 | it_++; 51 | return *this; 52 | } 53 | 54 | iterator &operator--() { 55 | it_--; 56 | return *this; 57 | } 58 | 59 | bool operator==(const iterator &rhs) { 60 | return it_ == rhs.it_; 61 | } 62 | bool operator!=(const iterator &rhs) { 63 | return !(*this == rhs); 64 | } 65 | protected: 66 | iterator(chunk_allocator *alloc, piece *it) 67 | : alloc_(alloc), it_(it) {} 68 | 69 | chunk_allocator *alloc_; 70 | piece *it_; 71 | 72 | friend class file_contents; 73 | }; 74 | 75 | file_contents(uint32_t npieces) : npieces_(npieces) { } 76 | 77 | iterator begin(chunk_allocator *alloc) { 78 | return iterator(alloc, pieces_); 79 | } 80 | 81 | iterator end(chunk_allocator *alloc) { 82 | return iterator(alloc, pieces_ + npieces_); 83 | } 84 | 85 | piece *begin() { 86 | return pieces_; 87 | } 88 | 89 | piece *end() { 90 | return pieces_ + npieces_; 91 | } 92 | 93 | size_t size() { 94 | return npieces_; 95 | } 96 | 97 | friend class codesearch_index; 98 | friend class load_allocator; 99 | friend class file_contents_builder; 100 | 101 | protected: 102 | file_contents() {} 103 | 104 | uint32_t npieces_; 105 | piece pieces_[]; 106 | }; 107 | 108 | class file_contents_builder { 109 | public: 110 | void extend(chunk *chunk, const StringPiece &piece); 111 | file_contents *build(chunk_allocator *alloc); 112 | protected: 113 | vector pieces_; 114 | }; 115 | 116 | #endif 117 | -------------------------------------------------------------------------------- /src/dump_load.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- dump_load.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_DUMP_LOAD_H 9 | #define CODESEARCH_DUMP_LOAD_H 10 | 11 | #include 12 | 13 | const uint32_t kIndexMagic = 0xc0d35eac; 14 | const uint32_t kIndexVersion = 13; 15 | const uint32_t kPageSize = (1 << 12); 16 | 17 | struct index_header { 18 | uint32_t magic; 19 | uint32_t version; 20 | uint32_t chunk_size; 21 | 22 | uint64_t name_off; 23 | 24 | uint32_t ntrees; 25 | uint64_t refs_off; 26 | 27 | uint32_t nfiles; 28 | uint64_t files_off; 29 | 30 | uint32_t nchunks; 31 | uint64_t chunks_off; 32 | 33 | uint32_t ncontent; 34 | uint64_t content_off; 35 | } __attribute__((packed)); 36 | 37 | struct chunk_header { 38 | uint64_t data_off; 39 | uint64_t files_off; 40 | uint32_t size; 41 | uint32_t nfiles; 42 | } __attribute__((packed)); 43 | 44 | struct content_chunk_header { 45 | uint64_t file_off; 46 | uint32_t size; 47 | } __attribute__((packed)); 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /src/fs_indexer.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "src/lib/recursion.h" 7 | 8 | #include "src/codesearch.h" 9 | #include "src/fs_indexer.h" 10 | #include 11 | 12 | static int kMaxRecursion = 100; 13 | 14 | using namespace std; 15 | namespace fs = boost::filesystem; 16 | 17 | fs_indexer::fs_indexer(code_searcher *cs, 18 | const string& repopath, 19 | const string& name, 20 | const Metadata &metadata) 21 | : cs_(cs), repopath_(repopath), name_(name) { 22 | tree_ = cs->open_tree(name, metadata, ""); 23 | } 24 | 25 | fs_indexer::~fs_indexer() { 26 | } 27 | 28 | void fs_indexer::read_file(const fs::path& path) { 29 | ifstream in(path.c_str(), ios::in); 30 | fs::path relpath = fs::relative(path, repopath_); 31 | cs_->index_file(tree_, relpath.string(), StringPiece(static_cast(stringstream() << in.rdbuf()).str().c_str(), fs::file_size(path))); 32 | } 33 | 34 | void fs_indexer::walk_contents_file(const fs::path& contents_file_path) { 35 | ifstream contents_file(contents_file_path.c_str(), ios::in); 36 | if (!contents_file.is_open()) { 37 | throw std::ifstream::failure("Unable to open contents file for reading: " + contents_file_path.string()); 38 | } 39 | string path; 40 | while (std::getline(contents_file, path)) { 41 | if (path.length()) { 42 | read_file(fs::path(repopath_) / path); 43 | } 44 | } 45 | } 46 | 47 | void fs_indexer::walk(const fs::path& path) { 48 | static int recursion_depth = 0; 49 | RecursionCounter guard(recursion_depth); 50 | if (recursion_depth > kMaxRecursion) 51 | return; 52 | if (!fs::exists(path)) return; 53 | fs::directory_iterator end_itr; 54 | if (fs::is_directory(path)) { 55 | for (fs::directory_iterator itr(path); 56 | itr != end_itr; 57 | ++itr) { 58 | if (fs::is_directory(itr->status()) ) { 59 | fs_indexer::walk(itr->path()); 60 | } else if (fs::is_regular_file(itr->status()) ) { 61 | fs_indexer::read_file(itr->path()); 62 | } 63 | } 64 | } else if (fs::is_regular_file(path)) { 65 | fs_indexer::read_file(path); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/fs_indexer.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- fs_indexer.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_FS_INDEXER_H 9 | #define CODESEARCH_FS_INDEXER_H 10 | 11 | #include 12 | #include "src/proto/config.pb.h" 13 | 14 | class code_searcher; 15 | struct indexed_tree; 16 | namespace boost { namespace filesystem { class path; } } 17 | 18 | 19 | class fs_indexer { 20 | public: 21 | fs_indexer(code_searcher *cs, 22 | const string& repopath, 23 | const string& name, 24 | const Metadata &metadata); 25 | ~fs_indexer(); 26 | void walk(const boost::filesystem::path& path); 27 | void walk_contents_file(const boost::filesystem::path& contents_file_path); 28 | protected: 29 | code_searcher *cs_; 30 | std::string repopath_; 31 | std::string name_; 32 | const indexed_tree *tree_; 33 | 34 | void read_file(const boost::filesystem::path& path); 35 | }; 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /src/git_indexer.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "src/lib/metrics.h" 5 | #include "src/lib/debug.h" 6 | 7 | #include "src/codesearch.h" 8 | #include "src/git_indexer.h" 9 | #include "src/smart_git.h" 10 | 11 | #include "src/proto/config.pb.h" 12 | 13 | using namespace std; 14 | 15 | DEFINE_string(order_root, "", "Walk top-level directories in this order."); 16 | DEFINE_bool(revparse, false, "Display parsed revisions, rather than as-provided"); 17 | 18 | git_indexer::git_indexer(code_searcher *cs, 19 | const string& repopath, 20 | const string& name, 21 | const Metadata &metadata) 22 | : cs_(cs), repo_(0), name_(name), metadata_(metadata) { 23 | int err; 24 | if ((err = git_libgit2_init()) < 0) 25 | die("git_libgit2_init: %s", giterr_last()->message); 26 | 27 | git_repository_open(&repo_, repopath.c_str()); 28 | if (repo_ == NULL) { 29 | fprintf(stderr, "Unable to open repo: %s\n", repopath.c_str()); 30 | exit(1); 31 | } 32 | git_libgit2_opts(GIT_OPT_SET_CACHE_OBJECT_LIMIT, GIT_OBJ_BLOB, 10*1024); 33 | git_libgit2_opts(GIT_OPT_SET_CACHE_OBJECT_LIMIT, GIT_OBJ_OFS_DELTA, 10*1024); 34 | git_libgit2_opts(GIT_OPT_SET_CACHE_OBJECT_LIMIT, GIT_OBJ_REF_DELTA, 10*1024); 35 | } 36 | 37 | git_indexer::~git_indexer() { 38 | git_repository_free(repo_); 39 | } 40 | 41 | void git_indexer::walk(const string& ref) { 42 | smart_object commit; 43 | smart_object tree; 44 | if (0 != git_revparse_single(commit, repo_, (ref + "^0").c_str())) { 45 | fprintf(stderr, "ref %s not found, skipping (empty repo?)\n", ref.c_str()); 46 | return; 47 | } 48 | git_commit_tree(tree, commit); 49 | 50 | char oidstr[GIT_OID_HEXSZ+1]; 51 | string version = FLAGS_revparse ? 52 | strdup(git_oid_tostr(oidstr, sizeof(oidstr), git_commit_id(commit))) : ref; 53 | 54 | idx_tree_ = cs_->open_tree(name_, metadata_, version); 55 | walk_tree("", FLAGS_order_root, tree); 56 | } 57 | 58 | void git_indexer::walk_tree(const string& pfx, 59 | const string& order, 60 | git_tree *tree) { 61 | map root; 62 | vector ordered; 63 | int entries = git_tree_entrycount(tree); 64 | for (int i = 0; i < entries; ++i) { 65 | const git_tree_entry *ent = git_tree_entry_byindex(tree, i); 66 | root[git_tree_entry_name(ent)] = ent; 67 | } 68 | 69 | istringstream stream(order); 70 | string dir; 71 | while(stream >> dir) { 72 | map::iterator it = root.find(dir); 73 | if (it == root.end()) 74 | continue; 75 | ordered.push_back(it->second); 76 | root.erase(it); 77 | } 78 | for (map::iterator it = root.begin(); 79 | it != root.end(); ++it) 80 | ordered.push_back(it->second); 81 | for (vector::iterator it = ordered.begin(); 82 | it != ordered.end(); ++it) { 83 | smart_object obj; 84 | git_tree_entry_to_object(obj, repo_, *it); 85 | string path = pfx + git_tree_entry_name(*it); 86 | 87 | if (git_tree_entry_type(*it) == GIT_OBJ_TREE) { 88 | walk_tree(path + "/", "", obj); 89 | } else if (git_tree_entry_type(*it) == GIT_OBJ_BLOB) { 90 | const char *data = static_cast(git_blob_rawcontent(obj)); 91 | cs_->index_file(idx_tree_, path, StringPiece(data, git_blob_rawsize(obj))); 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/git_indexer.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- git_indexer.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_GIT_INDEXER_H 9 | #define CODESEARCH_GIT_INDEXER_H 10 | 11 | #include 12 | #include "src/proto/config.pb.h" 13 | 14 | class code_searcher; 15 | class git_repository; 16 | class git_tree; 17 | struct indexed_tree; 18 | 19 | class git_indexer { 20 | public: 21 | git_indexer(code_searcher *cs, 22 | const std::string& repopath, 23 | const std::string& name, 24 | const Metadata &metadata); 25 | ~git_indexer(); 26 | void walk(const std::string& ref); 27 | protected: 28 | void walk_tree(const std::string& pfx, 29 | const std::string& order, 30 | git_tree *tree); 31 | 32 | code_searcher *cs_; 33 | git_repository *repo_; 34 | const indexed_tree *idx_tree_; 35 | std::string name_; 36 | Metadata metadata_; 37 | }; 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /src/lib/BUILD: -------------------------------------------------------------------------------- 1 | cc_library( 2 | name = "lib", 3 | srcs = glob(["*.cc"]), 4 | hdrs = glob(["*.h"]), 5 | copts = ["-Wno-sign-compare"], 6 | visibility = ["//visibility:public"], 7 | deps = [ 8 | "@com_github_gflags_gflags//:gflags", 9 | ], 10 | ) 11 | -------------------------------------------------------------------------------- /src/lib/debug.cc: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- debug.cc 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #include "debug.h" 9 | 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "per_thread.h" 22 | 23 | using std::string; 24 | 25 | debug_mode debug_enabled; 26 | 27 | DEFINE_string(debug, "", "Enable debugging for selected subsystems"); 28 | 29 | static per_thread trace_id; 30 | 31 | struct debug_flag { 32 | const char *flag; 33 | debug_mode bits; 34 | } debug_flags[] = { 35 | {"search", kDebugSearch}, 36 | {"profile", kDebugProfile}, 37 | {"index", kDebugIndex}, 38 | {"indexall", kDebugIndexAll}, 39 | {"ui", kDebugUI}, 40 | {"all", (debug_mode)-1} 41 | }; 42 | 43 | static bool validate_debug(const char *flagname, const string& value) { 44 | off_t off = 0; 45 | while (off < value.size()) { 46 | string opt; 47 | off_t comma = value.find(',', off); 48 | if (comma == string::npos) 49 | comma = value.size(); 50 | opt = value.substr(off, comma - off); 51 | off = comma + 1; 52 | 53 | bool found = false; 54 | for (int i = 0; i < sizeof(debug_flags)/sizeof(*debug_flags); ++i) { 55 | if (opt == debug_flags[i].flag) { 56 | found = true; 57 | debug_enabled = static_cast(debug_enabled | debug_flags[i].bits); 58 | break; 59 | } 60 | } 61 | 62 | if (!found) { 63 | return false; 64 | } 65 | } 66 | 67 | return true; 68 | } 69 | 70 | static const bool dummy = gflags::RegisterFlagValidator(&FLAGS_debug, 71 | validate_debug); 72 | 73 | 74 | string vstrprintf(const char *fmt, va_list ap) { 75 | char *buf = NULL; 76 | int err = vasprintf(&buf, fmt, ap); 77 | if (err <= 0) { 78 | fprintf(stderr, "unable to log: fmt='%s' err=%s\n", 79 | fmt, strerror(errno)); 80 | return ""; 81 | } 82 | 83 | string out(buf, err); 84 | free(buf); 85 | return out; 86 | } 87 | 88 | string strprintf(const char *fmt, ...) { 89 | va_list ap; 90 | va_start(ap, fmt); 91 | string out = vstrprintf(fmt, ap); 92 | va_end(ap); 93 | return out; 94 | } 95 | 96 | void cs_debug(const char *file, int lno, const char *fmt, ...) { 97 | va_list ap; 98 | va_start(ap, fmt); 99 | 100 | string buf; 101 | if (current_trace_id().empty()) 102 | buf = strprintf("[%s:%d] %s\n", 103 | file, lno, vstrprintf(fmt, ap).c_str()); 104 | else 105 | buf = strprintf("[%s][%s:%d] %s\n", 106 | trace_id->c_str(), file, lno, vstrprintf(fmt, ap).c_str()); 107 | 108 | va_end(ap); 109 | 110 | fputs(buf.c_str(), stderr); 111 | } 112 | 113 | 114 | void die(const char *fmt, ...) { 115 | va_list ap; 116 | va_start(ap, fmt); 117 | vfprintf(stderr, fmt, ap); 118 | fprintf(stderr, "\n"); 119 | va_end(ap); 120 | exit(1); 121 | } 122 | 123 | void vlog(const std::string &trace, const char *fmt, va_list ap) { 124 | auto now_time_point = std::chrono::system_clock::now(); 125 | auto now_time_t = std::chrono::system_clock::to_time_t(now_time_point); 126 | struct tm now_tm; 127 | ::gmtime_r(&now_time_t, &now_tm); 128 | 129 | char datestr[32]; 130 | strftime(datestr, sizeof(datestr), "%Y-%m-%d %H:%M:%S", &now_tm); 131 | 132 | string buf; 133 | if (trace.empty()) 134 | buf = vstrprintf(fmt, ap); 135 | else 136 | buf = strprintf("[%s] %s", 137 | trace.c_str(), vstrprintf(fmt, ap).c_str()); 138 | 139 | fprintf(stderr, "%s %s\n", datestr, buf.c_str()); 140 | } 141 | 142 | void log(const std::string &trace, const char *fmt, ...) { 143 | va_list ap; 144 | va_start(ap, fmt); 145 | vlog(trace, fmt, ap); 146 | va_end(ap); 147 | } 148 | 149 | void log(const char *fmt, ...) { 150 | va_list ap; 151 | va_start(ap, fmt); 152 | vlog(current_trace_id(), fmt, ap); 153 | va_end(ap); 154 | } 155 | 156 | std::string current_trace_id() { 157 | if (trace_id.get() == nullptr) 158 | trace_id.put(new std::string()); 159 | return *trace_id.get(); 160 | } 161 | 162 | scoped_trace_id::scoped_trace_id(const std::string &tid) { 163 | orig_ = trace_id.put(new std::string(tid)); 164 | } 165 | 166 | scoped_trace_id::~scoped_trace_id() { 167 | delete trace_id.put(orig_); 168 | } 169 | -------------------------------------------------------------------------------- /src/lib/debug.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- debug.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_DEBUG_H 9 | #define CODESEARCH_DEBUG_H 10 | 11 | #include 12 | 13 | enum debug_mode { 14 | kDebugSearch = 0x0001, 15 | kDebugProfile = 0x0002, 16 | kDebugIndex = 0x0004, 17 | kDebugIndexAll = 0x0008, 18 | kDebugUI = 0x0010, 19 | }; 20 | 21 | extern debug_mode debug_enabled; 22 | 23 | #define debug(which, ...) do { \ 24 | if (debug_enabled & (which)) \ 25 | cs_debug(__FILE__, __LINE__, ##__VA_ARGS__); \ 26 | } while (0) \ 27 | 28 | void cs_debug(const char *file, int lno, const char *fmt, ...) 29 | __attribute__((format (printf, 3, 4))); 30 | 31 | std::string strprintf(const char *fmt, ...) 32 | __attribute__((format (printf, 1, 2))); 33 | 34 | void die(const char *fmt, ...) 35 | __attribute__((format (printf, 1, 2), noreturn)); 36 | 37 | void log(const char *fmt, ...) 38 | __attribute__((format (printf, 1, 2))); 39 | void log(const std::string &trace, const char *fmt, ...) 40 | __attribute__((format (printf, 2, 3))); 41 | 42 | std::string current_trace_id(); 43 | 44 | class scoped_trace_id { 45 | public: 46 | scoped_trace_id(const std::string &tid); 47 | ~scoped_trace_id(); 48 | private: 49 | std::string *orig_; 50 | }; 51 | 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /src/lib/metrics.cc: -------------------------------------------------------------------------------- 1 | #include "metrics.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace { 8 | std::mutex metrics_mtx; 9 | std::map *metrics; 10 | }; 11 | 12 | 13 | metric::metric(const std::string &name) { 14 | std::unique_lock locked(metrics_mtx); 15 | if (metrics == 0) 16 | metrics = new std::map; 17 | (*metrics)[name] = this; 18 | } 19 | 20 | 21 | void metric::dump_all() { 22 | fprintf(stderr, "== begin metrics ==\n"); 23 | for (auto it = metrics->begin(); it != metrics->end(); ++it) { 24 | fprintf(stderr, "%s %ld\n", it->first.c_str(), it->second->val_.load()); 25 | } 26 | fprintf(stderr, "== end metrics ==\n"); 27 | } 28 | -------------------------------------------------------------------------------- /src/lib/metrics.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- metrics.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_METRICS_H 9 | #define CODESEARCH_METRICS_H 10 | 11 | #include "timer.h" 12 | 13 | #include 14 | #include 15 | 16 | class metric { 17 | public: 18 | metric(const std::string &name); 19 | void inc() {++val_;} 20 | void inc(long i) {val_ += i;} 21 | void dec() {--val_;} 22 | void dec(long i) {val_ -= i;} 23 | 24 | static void dump_all(); 25 | 26 | #ifdef CODESEARCH_SLOWGTOD 27 | class timer { 28 | public: 29 | timer(metric &m) {} 30 | void pause() {}; 31 | void start() {}; 32 | }; 33 | #else 34 | class timer { 35 | public: 36 | timer(metric &m) : m_(&m) {} 37 | 38 | void pause() { 39 | tm_.pause(); 40 | timeval elapsed = tm_.elapsed(); 41 | m_->inc(elapsed.tv_sec * 1000000 + elapsed.tv_usec); 42 | tm_.reset(); 43 | } 44 | 45 | void start() { 46 | tm_.start(); 47 | } 48 | 49 | ~timer() { 50 | if (tm_.running()) 51 | pause(); 52 | } 53 | private: 54 | metric *m_; 55 | ::timer tm_; 56 | }; 57 | #endif 58 | 59 | private: 60 | std::atomic_long val_; 61 | }; 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /src/lib/per_thread.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- per_thread.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_PER_THREAD_H 9 | #define CODESEARCH_PER_THREAD_H 10 | 11 | #include 12 | 13 | template 14 | class per_thread { 15 | public: 16 | per_thread() { 17 | pthread_key_create(&key_, destroy); 18 | } 19 | ~per_thread() { 20 | pthread_key_delete(key_); 21 | } 22 | 23 | T *get() const { 24 | return static_cast(pthread_getspecific(key_)); 25 | } 26 | 27 | T *put(T *obj) { 28 | T *old = get(); 29 | pthread_setspecific(key_, obj); 30 | return old; 31 | } 32 | 33 | T *operator->() const { 34 | return get(); 35 | } 36 | 37 | T& operator*() const { 38 | return *get(); 39 | } 40 | private: 41 | 42 | static void destroy(void *obj) { 43 | T *t = static_cast(obj); 44 | delete t; 45 | } 46 | 47 | pthread_key_t key_; 48 | 49 | per_thread(const per_thread& rhs); 50 | void operator=(const per_thread& rhs); 51 | }; 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /src/lib/radix_sort.cc: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- radix_sort.cc 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #include 9 | #include 10 | #include 11 | 12 | using std::vector; 13 | 14 | #include "per_thread.h" 15 | 16 | void lsd_radix_sort(uint32_t *left, uint32_t *right) 17 | { 18 | static per_thread > scratch; 19 | 20 | int width = right - left; 21 | if (!scratch.get()) 22 | scratch.put(new vector(width)); 23 | scratch->reserve(width); 24 | uint32_t *cur = left, *other = &(*scratch)[0]; 25 | uint32_t counts[4][256]; 26 | /* 27 | * We do four passes 28 | * (0) input -> scratch 29 | * (1) scratch -> input 30 | * (2) input -> scratch 31 | * (3) scratch -> input 32 | * 33 | * So after the fourth pass, the input array is sorted and back in 34 | * the original storage. 35 | */ 36 | 37 | memset(counts, 0, sizeof counts); 38 | for (int i = 0; i < width; i++) { 39 | counts[0][(cur[i] >> 0 ) & 0xFF]++; 40 | counts[1][(cur[i] >> 8 ) & 0xFF]++; 41 | counts[2][(cur[i] >> 16) & 0xFF]++; 42 | counts[3][(cur[i] >> 24) & 0xFF]++; 43 | } 44 | 45 | for (int digit = 0; digit < 4; digit++) { 46 | int total = 0; 47 | for (int i = 0; i < 256; i++) { 48 | int tmp = counts[digit][i]; 49 | counts[digit][i] = total; 50 | total += tmp; 51 | } 52 | for (int i = 0; i < width; i++) { 53 | int d = (cur[i] >> (8 * digit)) & 0xFF; 54 | other[counts[digit][d]++] = cur[i]; 55 | } 56 | uint32_t *tmp = cur; 57 | cur = other; 58 | other = tmp; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/lib/radix_sort.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- radix_sort.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_RADIX_SORT_H 9 | #define CODESEARCH_RADIX_SORT_H 10 | 11 | #include 12 | #include 13 | 14 | void lsd_radix_sort(uint32_t *left, uint32_t *right); 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /src/lib/recursion.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- recursion.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_RECURSION_H 9 | #define CODESEARCH_RECURSION_H 10 | 11 | class RecursionCounter { 12 | public: 13 | RecursionCounter(int &depth) : depth_(depth) { 14 | depth_++; 15 | } 16 | ~RecursionCounter() { 17 | depth_--; 18 | } 19 | protected: 20 | int &depth_; 21 | }; 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /src/lib/thread_queue.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- thread_queue.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_THREAD_QUEUE_H 9 | #define CODESEARCH_THREAD_QUEUE_H 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | template 16 | class thread_queue { 17 | public: 18 | thread_queue () : closed_(false) {} 19 | 20 | void push(const T& val) { 21 | std::unique_lock locked(mutex_); 22 | queue_.push_back(val); 23 | cond_.notify_one(); 24 | } 25 | 26 | void close() { 27 | std::unique_lock locked(mutex_); 28 | closed_ = true; 29 | cond_.notify_all(); 30 | } 31 | 32 | bool pop(T *out) { 33 | std::unique_lock locked(mutex_); 34 | while (queue_.empty() && !closed_) 35 | cond_.wait(locked); 36 | if (queue_.empty() && closed_) 37 | return false; 38 | *out = queue_.front(); 39 | queue_.pop_front(); 40 | return true; 41 | } 42 | 43 | bool try_pop(T *out) { 44 | std::unique_lock locked(mutex_); 45 | if (queue_.empty()) 46 | return false; 47 | *out = queue_.front(); 48 | queue_.pop_front(); 49 | return true; 50 | } 51 | 52 | protected: 53 | thread_queue(const thread_queue&); 54 | thread_queue operator=(const thread_queue &); 55 | std::mutex mutex_; 56 | std::condition_variable cond_; 57 | bool closed_; 58 | std::list queue_; 59 | }; 60 | 61 | 62 | #endif /* CODESEARCH_THREAD_QUEUE_H */ 63 | -------------------------------------------------------------------------------- /src/lib/timer.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- timer.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_TIMER_H 9 | #define CODESEARCH_TIMER_H 10 | #include 11 | #include 12 | #include 13 | 14 | static int timeval_subtract (struct timeval *result, struct timeval *x, struct timeval *y); 15 | static void timeval_add(struct timeval *res, const struct timeval *x, const struct timeval *y); 16 | 17 | class timer { 18 | public: 19 | timer(bool startnow = true) 20 | : running_(false), elapsed_({0,0}){ 21 | if (startnow) 22 | start(); 23 | } 24 | 25 | void start() { 26 | std::unique_lock locked(lock_); 27 | assert(!running_); 28 | running_ = true; 29 | gettimeofday(&start_, NULL); 30 | } 31 | 32 | void pause() { 33 | std::unique_lock locked(lock_); 34 | struct timeval now, delta; 35 | assert(running_); 36 | running_ = false; 37 | gettimeofday(&now, NULL); 38 | timeval_subtract(&delta, &now, &start_); 39 | timeval_add(&elapsed_, &delta, &elapsed_); 40 | } 41 | 42 | void reset() { 43 | std::unique_lock locked(lock_); 44 | running_ = false; 45 | elapsed_ = (struct timeval){0,0}; 46 | } 47 | 48 | void add(timer &other) { 49 | std::unique_lock locked(lock_); 50 | assert(!running_); 51 | struct timeval elapsed = other.elapsed(); 52 | timeval_add(&elapsed_, &elapsed_, &elapsed); 53 | } 54 | 55 | bool running() { 56 | std::unique_lock locked(lock_); 57 | return running_; 58 | } 59 | 60 | struct timeval elapsed() { 61 | std::unique_lock locked(lock_); 62 | if (running_) { 63 | struct timeval now, delta; 64 | gettimeofday(&now, NULL); 65 | timeval_subtract(&delta, &now, &start_); 66 | timeval_add(&elapsed_, &delta, &elapsed_); 67 | start_ = now; 68 | } 69 | return elapsed_; 70 | } 71 | 72 | protected: 73 | bool running_; 74 | struct timeval start_; 75 | struct timeval elapsed_; 76 | std::mutex lock_; 77 | 78 | timer(const timer& rhs); 79 | timer operator=(const timer& rhs); 80 | }; 81 | 82 | 83 | /* Subtract the `struct timeval' values X and Y, 84 | storing the result in RESULT. 85 | Return 1 if the difference is negative, otherwise 0. */ 86 | static inline int 87 | timeval_subtract (struct timeval *result, struct timeval *x, struct timeval *y) 88 | { 89 | /* Perform the carry for the later subtraction by updating y. */ 90 | if (x->tv_usec < y->tv_usec) { 91 | int nsec = (y->tv_usec - x->tv_usec) / 1000000 + 1; 92 | y->tv_usec -= 1000000 * nsec; 93 | y->tv_sec += nsec; 94 | } 95 | if (x->tv_usec - y->tv_usec > 1000000) { 96 | int nsec = (x->tv_usec - y->tv_usec) / 1000000; 97 | y->tv_usec += 1000000 * nsec; 98 | y->tv_sec -= nsec; 99 | } 100 | 101 | /* Compute the time remaining to wait. 102 | tv_usec is certainly positive. */ 103 | result->tv_sec = x->tv_sec - y->tv_sec; 104 | result->tv_usec = x->tv_usec - y->tv_usec; 105 | 106 | /* Return 1 if result is negative. */ 107 | return x->tv_sec < y->tv_sec; 108 | } 109 | 110 | static inline void 111 | timeval_add(struct timeval *res, const struct timeval *x, 112 | const struct timeval *y) 113 | { 114 | res->tv_sec = x->tv_sec + y->tv_sec; 115 | res->tv_usec = x->tv_usec + y->tv_usec; 116 | 117 | while (res->tv_usec > 1000000) { 118 | res->tv_usec -= 1000000; 119 | res->tv_sec++; 120 | } 121 | } 122 | 123 | class run_timer { 124 | public: 125 | run_timer(timer& timer) 126 | #ifndef CODESEARCH_SLOWGTOD 127 | : timer_(timer), local_() 128 | #endif 129 | { 130 | } 131 | ~run_timer() { 132 | #ifndef CODESEARCH_SLOWGTOD 133 | local_.pause(); 134 | timer_.add(local_); 135 | #endif 136 | } 137 | protected: 138 | #ifndef CODESEARCH_SLOWGTOD 139 | timer &timer_; 140 | timer local_; 141 | #endif 142 | }; 143 | 144 | inline static long timeval_ms(struct timeval tv) { 145 | return tv.tv_sec * 1000 + tv.tv_usec / 1000; 146 | } 147 | 148 | #endif 149 | -------------------------------------------------------------------------------- /src/proto/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") 4 | load("@build_stack_rules_proto//cpp:cpp_grpc_library.bzl", "cpp_grpc_library") 5 | 6 | proto_library( 7 | name = "livegrep_proto", 8 | srcs = ["livegrep.proto"], 9 | deps = [":config_proto"], 10 | ) 11 | 12 | proto_library( 13 | name = "config_proto", 14 | srcs = ["config.proto"], 15 | ) 16 | 17 | go_proto_library( 18 | name = "go_config_proto", 19 | importpath = "github.com/livegrep/livegrep/src/proto/config", 20 | proto = ":config_proto", 21 | ) 22 | 23 | go_proto_library( 24 | name = "go_proto", 25 | compilers = ["@io_bazel_rules_go//proto:go_grpc"], 26 | importpath = "github.com/livegrep/livegrep/src/proto/go_proto", 27 | proto = ":livegrep_proto", 28 | visibility = ["//visibility:public"], 29 | deps = [":go_config_proto"], 30 | ) 31 | 32 | cpp_grpc_library( 33 | name = "cc_proto", 34 | deps = [":livegrep_proto"], 35 | ) 36 | 37 | cc_proto_library( 38 | name = "cc_config_proto", 39 | deps = [":config_proto"], 40 | ) 41 | -------------------------------------------------------------------------------- /src/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | message IndexSpec { 4 | string name = 1; 5 | repeated PathSpec paths = 2 [json_name = "fs_paths"]; 6 | repeated RepoSpec repos = 3 [json_name = "repositories"]; 7 | } 8 | 9 | message Metadata { 10 | string url_pattern = 1 [json_name = "url-pattern"]; 11 | string remote = 2 [json_name = "remote"]; 12 | string github = 3 [json_name = "github"]; 13 | } 14 | 15 | message PathSpec { 16 | string path = 1 [json_name = "path"]; 17 | string name = 2 [json_name = "name"]; 18 | string ordered_contents = 3 [json_name = "ordered-contents"]; 19 | Metadata metadata = 4 [json_name = "metadata"]; 20 | } 21 | 22 | message RepoSpec { 23 | string path = 1 [json_name = "path"]; 24 | string name = 2 [json_name = "name"]; 25 | repeated string revisions = 3 [json_name = "revisions"]; 26 | Metadata metadata = 4 [json_name = "metadata"]; 27 | } 28 | -------------------------------------------------------------------------------- /src/proto/livegrep.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "src/proto/config.proto"; 4 | 5 | message Query { 6 | string line = 1; 7 | string file = 2; 8 | string repo = 3; 9 | string tags = 4; 10 | bool fold_case = 5; 11 | string not_file = 6; 12 | string not_repo = 7; 13 | string not_tags = 8; 14 | int32 max_matches = 9; 15 | bool filename_only = 10; 16 | } 17 | 18 | message Bounds { 19 | int32 left = 1; 20 | int32 right = 2; 21 | } 22 | 23 | message SearchResult { 24 | string tree = 1; 25 | string version = 2; 26 | string path = 3; 27 | int64 line_number = 4; 28 | repeated string context_before = 5; 29 | repeated string context_after = 6; 30 | Bounds bounds = 7; 31 | string line = 8; 32 | } 33 | 34 | message FileResult { 35 | string tree = 1; 36 | string version = 2; 37 | string path = 3; 38 | Bounds bounds = 4; 39 | } 40 | 41 | message SearchStats { 42 | int64 re2_time = 1; 43 | int64 git_time = 2; 44 | int64 sort_time = 3; 45 | int64 index_time = 4; 46 | int64 analyze_time = 5; 47 | enum ExitReason { 48 | NONE = 0; 49 | TIMEOUT = 1; 50 | MATCH_LIMIT = 2; 51 | } 52 | ExitReason exit_reason = 6; 53 | } 54 | 55 | message ServerInfo { 56 | string name = 1; 57 | message Tree { 58 | string name = 1; 59 | string version = 2; 60 | Metadata metadata = 3; 61 | } 62 | repeated Tree trees = 2; 63 | bool has_tags = 3; 64 | // unix timestamp (seconds) 65 | int64 index_time = 4; 66 | } 67 | 68 | message CodeSearchResult { 69 | SearchStats stats = 1; 70 | repeated SearchResult results = 2; 71 | repeated FileResult file_results = 3; 72 | } 73 | 74 | message InfoRequest { 75 | } 76 | 77 | message Empty { 78 | } 79 | 80 | service CodeSearch { 81 | rpc Info(InfoRequest) returns (ServerInfo); 82 | rpc Search(Query) returns (CodeSearchResult); 83 | rpc Reload(Empty) returns (Empty); 84 | } 85 | -------------------------------------------------------------------------------- /src/query_planner.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- query_planner.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_INDEXER_H 9 | #define CODESEARCH_INDEXER_H 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "re2/re2.h" 20 | #include "re2/walker-inl.h" 21 | 22 | #include "src/common.h" 23 | 24 | using std::string; 25 | using std::vector; 26 | using std::list; 27 | using std::set; 28 | using boost::intrusive_ptr; 29 | 30 | enum { 31 | kAnchorNone = 0x00, 32 | kAnchorLeft = 0x01, 33 | kAnchorRight = 0x02, 34 | kAnchorBoth = 0x03, 35 | kAnchorRepeat = 0x04 36 | }; 37 | 38 | class QueryPlan { 39 | public: 40 | typedef std::map, intrusive_ptr >::iterator iterator; 41 | typedef std::map, intrusive_ptr >::const_iterator const_iterator; 42 | typedef std::pair, intrusive_ptr > value_type; 43 | 44 | iterator begin() { 45 | return edges_.begin(); 46 | } 47 | 48 | iterator end() { 49 | return edges_.end(); 50 | } 51 | 52 | QueryPlan(int anchor = kAnchorNone) 53 | : anchor(anchor), refs_(0) { } 54 | 55 | QueryPlan(std::pair p, 56 | intrusive_ptr next, 57 | int anchor = kAnchorNone) 58 | : anchor(anchor), refs_(0) { 59 | insert(value_type(p, next)); 60 | } 61 | 62 | void insert(const value_type& v); 63 | void concat(intrusive_ptr rhs); 64 | 65 | bool empty() { 66 | return edges_.empty(); 67 | } 68 | 69 | size_t size() { 70 | return edges_.size(); 71 | } 72 | 73 | class Stats { 74 | public: 75 | double selectivity_; 76 | int depth_; 77 | long nodes_; 78 | long tail_paths_; 79 | 80 | Stats(); 81 | Stats insert(const value_type& v) const; 82 | Stats concat(const Stats& rhs) const; 83 | }; 84 | 85 | const Stats& stats() { 86 | return stats_; 87 | } 88 | 89 | /* 90 | * Returns an approximation of the fraction of the input corpus 91 | * that this index key will reduce the search space to. 92 | * 93 | * e.g. selectivity() == 1.0 implies that this index key includes 94 | * the entire input. 95 | * 96 | * selectivity() == 0.1 means that using this index key will 97 | * only require searching 1/10th of the corpus. 98 | * 99 | * This value is computed without any reference to the actual 100 | * characteristics of any particular corpus, and so is a rough 101 | * approximation at best. 102 | */ 103 | double selectivity(); 104 | 105 | /* 106 | * Returns a value approximating the "goodness" of this index key, 107 | * in arbitrary units. Higher is better. The weight incorporates 108 | * both the selectivity, above, and the cost of using this index 109 | * key. 110 | */ 111 | unsigned weight(); 112 | int depth(); 113 | long nodes(); 114 | 115 | string ToString(); 116 | 117 | void check_rep(); 118 | 119 | int anchor; 120 | 121 | void collect_tails(list& tails); 122 | protected: 123 | std::map, intrusive_ptr > edges_; 124 | Stats stats_; 125 | std::atomic_int refs_; 126 | 127 | void collect_tails(list& tails); 128 | void collect_tails(list& tails, 129 | set &seen); 130 | 131 | private: 132 | QueryPlan(const QueryPlan&); 133 | void operator=(const QueryPlan&); 134 | 135 | friend void intrusive_ptr_add_ref(QueryPlan *key); 136 | friend void intrusive_ptr_release(QueryPlan *key); 137 | }; 138 | 139 | intrusive_ptr constructQueryPlan(const re2::RE2 &pat); 140 | 141 | #endif /* CODESEARCH_INDEXER_H */ 142 | -------------------------------------------------------------------------------- /src/re_width.cc: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- re_width.cc 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #include "re_width.h" 9 | 10 | using namespace re2; 11 | 12 | int WidthWalker::ShortVisit(Regexp *re, int parent_arg) { 13 | return 0; 14 | } 15 | 16 | int WidthWalker::PostVisit(Regexp *re, int parent_arg, 17 | int pre_arg, 18 | int *child_args, int nchild_args) { 19 | int width; 20 | switch (re->op()) { 21 | case kRegexpRepeat: 22 | width = child_args[0] * re->max(); 23 | break; 24 | 25 | case kRegexpNoMatch: 26 | // These ops match the empty string: 27 | case kRegexpEmptyMatch: // anywhere 28 | case kRegexpBeginLine: // at beginning of line 29 | case kRegexpEndLine: // at end of line 30 | case kRegexpBeginText: // at beginning of text 31 | case kRegexpEndText: // at end of text 32 | case kRegexpWordBoundary: // at word boundary 33 | case kRegexpNoWordBoundary: // not at word boundary 34 | width = 0; 35 | break; 36 | 37 | case kRegexpLiteral: 38 | case kRegexpAnyChar: 39 | case kRegexpAnyByte: 40 | case kRegexpCharClass: 41 | width = 1; 42 | break; 43 | 44 | case kRegexpLiteralString: 45 | width = re->nrunes(); 46 | break; 47 | 48 | case kRegexpConcat: 49 | width = 0; 50 | for (int i = 0; i < nchild_args; i++) 51 | width += child_args[i]; 52 | break; 53 | 54 | case kRegexpAlternate: 55 | width = 0; 56 | for (int i = 0; i < nchild_args; i++) 57 | width = std::max(width, child_args[i]); 58 | break; 59 | 60 | case kRegexpStar: 61 | case kRegexpPlus: 62 | case kRegexpQuest: 63 | case kRegexpCapture: 64 | width = child_args[0]; 65 | break; 66 | 67 | default: 68 | abort(); 69 | } 70 | 71 | return width; 72 | } 73 | -------------------------------------------------------------------------------- /src/re_width.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- re_width.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_RE_WIDTH_H 9 | #define CODESEARCH_RE_WIDTH_H 10 | 11 | #include "re2/regexp.h" 12 | #include "re2/walker-inl.h" 13 | 14 | using re2::Regexp; 15 | 16 | class WidthWalker : public Regexp::Walker { 17 | public: 18 | virtual int PostVisit( 19 | Regexp* re, int parent_arg, 20 | int pre_arg, 21 | int *child_args, int nchild_args); 22 | 23 | virtual int ShortVisit( 24 | Regexp* re, 25 | int parent_arg); 26 | }; 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /src/smart_git.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- smart_git.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef CODESEARCH_SMART_GIT_H 9 | #define CODESEARCH_SMART_GIT_H 10 | 11 | #include "git2.h" 12 | 13 | class smart_object_base { 14 | public: 15 | smart_object_base() : obj_(0) { 16 | }; 17 | 18 | operator git_object* () { 19 | return obj_; 20 | } 21 | 22 | operator git_object** () { 23 | return &obj_; 24 | } 25 | 26 | ~smart_object_base() { 27 | if (obj_) 28 | git_object_free(obj_); 29 | } 30 | 31 | git_object *release() { 32 | git_object *o = obj_; 33 | obj_ = 0; 34 | return o; 35 | } 36 | 37 | protected: 38 | smart_object_base(const smart_object_base& rhs) { 39 | } 40 | git_object *obj_; 41 | }; 42 | 43 | template 44 | class object_traits { const static git_otype type; }; 45 | 46 | template <> 47 | struct object_traits { const static git_otype git_type = GIT_OBJ_TREE; }; 48 | template <> 49 | struct object_traits { const static git_otype git_type = GIT_OBJ_COMMIT; }; 50 | template <> 51 | struct object_traits { const static git_otype git_type = GIT_OBJ_BLOB; }; 52 | template <> 53 | struct object_traits { const static git_otype git_type = GIT_OBJ_TAG; }; 54 | 55 | template <> 56 | struct object_traits { const static git_otype git_type = GIT_OBJ_TREE; }; 57 | template <> 58 | struct object_traits { const static git_otype git_type = GIT_OBJ_COMMIT; }; 59 | template <> 60 | struct object_traits { const static git_otype git_type = GIT_OBJ_BLOB; }; 61 | template <> 62 | struct object_traits { const static git_otype git_type = GIT_OBJ_TAG; }; 63 | 64 | 65 | template 66 | class smart_object : public smart_object_base { 67 | public: 68 | operator T* () { 69 | assert(obj_); 70 | assert(git_object_type(obj_) == object_traits::git_type); 71 | return reinterpret_cast(obj_); 72 | } 73 | operator T** () { 74 | assert(obj_ == 0); 75 | return reinterpret_cast(&obj_); 76 | } 77 | 78 | T *release() { 79 | T *o = this; 80 | obj_ = 0; 81 | return o; 82 | } 83 | 84 | smart_object& operator=(git_object *rhs) { 85 | assert(obj_ == 0); 86 | assert(git_object_type(rhs) == object_traits::git_type); 87 | obj_ = rhs; 88 | return *this; 89 | } 90 | }; 91 | 92 | template <> 93 | class smart_object : public smart_object_base { 94 | public: 95 | template 96 | operator O* () { 97 | assert(git_object_type(obj_) == object_traits::git_type); 98 | return reinterpret_cast(obj_); 99 | } 100 | 101 | template 102 | operator O** () { 103 | assert(object_traits::git_type); 104 | return reinterpret_cast(&obj_); 105 | } 106 | 107 | }; 108 | 109 | #endif /* !defined(CODESEARCH_SMART_GIT_H) */ 110 | -------------------------------------------------------------------------------- /src/tagsearch.h: -------------------------------------------------------------------------------- 1 | /******************************************************************** 2 | * livegrep -- tagsearch.h 3 | * Copyright (c) 2011-2013 Nelson Elhage 4 | * 5 | * This program is free software. You may use, redistribute, and/or 6 | * modify it under the terms listed in the COPYING file. 7 | ********************************************************************/ 8 | #ifndef TAGSEARCH_H 9 | #define TAGSEARCH_H 10 | 11 | #include "src/codesearch.h" 12 | 13 | #include 14 | #include 15 | 16 | class chunk_allocator; 17 | 18 | class tag_searcher { 19 | public: 20 | void cache_indexed_files(code_searcher *cs); 21 | 22 | bool transform(query *q, match_result *m) const; 23 | 24 | static std::string create_tag_line_regex_from_query(query *q); 25 | 26 | protected: 27 | chunk_allocator *file_alloc_; 28 | std::map path_to_file_map_; 29 | }; 30 | 31 | #endif /* TAGSEARCH_H */ 32 | -------------------------------------------------------------------------------- /src/tools/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") 4 | 5 | cc_library( 6 | name = "grpc_server", 7 | srcs = [ 8 | "grpc_server.cc", 9 | "grpc_server.h", 10 | "limits.h", 11 | ], 12 | deps = [ 13 | "//src:codesearch", 14 | "//src/proto:cc_proto", 15 | "@boost//:bind", 16 | ], 17 | ) 18 | 19 | cc_binary( 20 | name = "codesearch", 21 | srcs = [ 22 | "codesearch.cc", 23 | "limits.h", 24 | ], 25 | copts = [ 26 | "-Wno-deprecated-declarations", 27 | "-Wno-sign-compare", 28 | ], 29 | deps = [ 30 | ":grpc_server", 31 | "//src:codesearch", 32 | "//src/proto:cc_config_proto", 33 | "//src/proto:cc_proto", 34 | "@boost//:bind", 35 | "@com_github_libgit2//:libgit2", 36 | ], 37 | ) 38 | 39 | cc_binary( 40 | name = "codesearchtool", 41 | srcs = [ 42 | "analyze-re.cc", 43 | "codesearchtool.cc", 44 | "dump-file.cc", 45 | "inspect-index.cc", 46 | ], 47 | copts = [ 48 | "-Wno-sign-compare", 49 | ], 50 | deps = [ 51 | "//src:codesearch", 52 | ], 53 | ) 54 | 55 | [genrule( 56 | name = "tool-" + t, 57 | srcs = [":codesearchtool"], 58 | outs = [t], 59 | cmd = "ln -nsf codesearchtool $@", 60 | output_to_bindir = 1, 61 | ) for t in [ 62 | "analyze-re", 63 | "dump-file", 64 | "inspect-index", 65 | ]] 66 | 67 | pkg_tar( 68 | name = "cc_tools", 69 | srcs = [ 70 | ":analyze-re", 71 | ":codesearch", 72 | ":codesearchtool", 73 | ":dump-file", 74 | ":grpc_server", 75 | ":inspect-index", 76 | ], 77 | package_dir = "bin/", 78 | ) 79 | -------------------------------------------------------------------------------- /src/tools/analyze-re.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | 15 | #include "src/lib/debug.h" 16 | 17 | #include "src/dump_load.h" 18 | #include "src/codesearch.h" 19 | #include "src/query_planner.h" 20 | #include "src/re_width.h" 21 | 22 | #include 23 | 24 | using namespace std; 25 | 26 | DEFINE_string(dot_index, "", "Write a graph of the index key as a dot graph."); 27 | DEFINE_bool(casefold, false, "Treat the regex as case-insensitive."); 28 | 29 | class QueryPlanDotOutputter { 30 | protected: 31 | map names_; 32 | set seen_; 33 | ofstream out_; 34 | int ct_; 35 | intrusive_ptr key_; 36 | 37 | string escape(char c) { 38 | if (c <= ' ' || c > '~' || c == '"' || c == '\\') 39 | return strprintf("\\\\x%hhx", c); 40 | return strprintf("%c", c); 41 | } 42 | 43 | void assign_names(intrusive_ptr key) { 44 | if (names_.find(key.get()) != names_.end()) 45 | return; 46 | names_[key.get()] = strprintf("node%d", ct_++); 47 | string flags; 48 | if (key->anchor & kAnchorLeft) 49 | flags += "^"; 50 | if (key->anchor & kAnchorRepeat) 51 | flags += "*"; 52 | if (key->anchor & kAnchorRight) 53 | flags += "$"; 54 | 55 | out_ << strprintf(" %s [label=\"%s\"]\n", 56 | names_[key.get()].c_str(), 57 | flags.c_str()); 58 | for (auto it = key->begin(); it != key->end(); it++) { 59 | if (!it->second) 60 | continue; 61 | assign_names(it->second); 62 | } 63 | } 64 | 65 | void dump(intrusive_ptr key) { 66 | if (seen_.find(key.get()) != seen_.end()) 67 | return; 68 | seen_.insert(key.get()); 69 | for (auto it = key->begin(); it != key->end(); it++) { 70 | string dst; 71 | if (!it->second) { 72 | out_ << strprintf(" node%d [shape=point,label=\"\"]\n", 73 | ct_); 74 | dst = strprintf(" node%d", ct_++); 75 | } else 76 | dst = names_[it->second.get()]; 77 | string label; 78 | if (it->first.first == it->first.second) 79 | label = escape(it->first.first); 80 | else 81 | label = strprintf("%s-%s", 82 | escape(it->first.first).c_str(), 83 | escape(it->first.second).c_str()); 84 | out_ << strprintf(" %s -> %s [label=\"%s\"]\n", 85 | names_[key.get()].c_str(), 86 | dst.c_str(), 87 | label.c_str()); 88 | if (it->second) 89 | dump(it->second); 90 | } 91 | } 92 | 93 | public: 94 | QueryPlanDotOutputter(const string &path, intrusive_ptr key) 95 | : out_(path.c_str()), ct_(0), key_(key) { 96 | } 97 | 98 | void output() { 99 | out_ << "digraph G {\n"; 100 | out_ << " rankdir=\"LR\"\n"; 101 | assign_names(key_); 102 | dump(key_); 103 | out_ << "}\n"; 104 | out_.close(); 105 | } 106 | }; 107 | 108 | 109 | void write_dot_index(const string &path, intrusive_ptr key) { 110 | QueryPlanDotOutputter out(path, key); 111 | out.output(); 112 | } 113 | 114 | int analyze_re(int argc, char **argv) { 115 | if (argc != 1) { 116 | fprintf(stderr, "Usage: %s REGEX\n", gflags::GetArgv0()); 117 | return 1; 118 | } 119 | 120 | RE2::Options opts; 121 | default_re2_options(opts); 122 | if (FLAGS_casefold) 123 | opts.set_case_sensitive(false); 124 | 125 | RE2 re(argv[0], opts); 126 | if (!re.ok()) { 127 | fprintf(stderr, "Error: %s\n", re.error().c_str()); 128 | return 1; 129 | } 130 | 131 | WidthWalker width; 132 | printf("== RE [%s] ==\n", argv[0]); 133 | printf("width: %d\n", width.Walk(re.Regexp(), 0)); 134 | printf("Program size: %d\n", re.ProgramSize()); 135 | 136 | intrusive_ptr key = constructQueryPlan(re); 137 | if (key) { 138 | QueryPlan::Stats stats = key->stats(); 139 | printf("Index key:\n"); 140 | printf(" log10(selectivity): %f\n", log(stats.selectivity_)/log(10)); 141 | printf(" depth: %d\n", stats.depth_); 142 | printf(" nodes: %ld\n", stats.nodes_); 143 | 144 | if (FLAGS_dot_index.size()) { 145 | write_dot_index(FLAGS_dot_index, key); 146 | } 147 | } else { 148 | printf("(Unindexable)\n"); 149 | } 150 | 151 | return 0; 152 | } 153 | -------------------------------------------------------------------------------- /src/tools/codesearchtool.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | using std::string; 7 | 8 | extern int analyze_re(int, char**); 9 | extern int dump_file(int, char**); 10 | extern int inspect_index(int, char**); 11 | 12 | struct _command { 13 | string name; 14 | int (*fn)(int, char**); 15 | } commands[] = { 16 | {"analyze-re", analyze_re}, 17 | {"inspect-index", inspect_index}, 18 | {"dump-file", dump_file}, 19 | }; 20 | 21 | int main(int argc, char **argv) { 22 | gflags::SetUsageMessage("Usage: " + string(argv[0]) + " COMMAND ARGS"); 23 | gflags::ParseCommandLineFlags(&argc, &argv, true); 24 | 25 | string me(argv[0]); 26 | size_t i = me.rfind('/'); 27 | if (i != string::npos) 28 | me = me.substr(i+1); 29 | 30 | for (auto it = &(commands[0]); 31 | it != &(commands[0]) + sizeof(commands)/sizeof(commands[0]); 32 | ++it) { 33 | if (it->name == me) 34 | return it->fn(argc-1, argv+1); 35 | } 36 | 37 | fprintf(stderr, "Unknown tool: %s\n", me.c_str()); 38 | 39 | return 1; 40 | } 41 | -------------------------------------------------------------------------------- /src/tools/dump-file.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | #include "src/lib/debug.h" 7 | 8 | #include "src/dump_load.h" 9 | #include "src/codesearch.h" 10 | #include "src/content.h" 11 | #include "src/re_width.h" 12 | 13 | #include 14 | 15 | void dump_file(code_searcher *cs, indexed_file *f) { 16 | for (auto it = f->content->begin(cs->alloc()); 17 | it != f->content->end(cs->alloc()); ++it) { 18 | printf("%.*s\n", (int)it->size(), it->data()); 19 | } 20 | } 21 | 22 | int dump_file(int argc, char **argv) { 23 | if (argc != 2) { 24 | fprintf(stderr, "Usage: %s INDEX PATH\n", gflags::GetArgv0()); 25 | return 1; 26 | } 27 | 28 | string index = argv[0]; 29 | string path = argv[1]; 30 | 31 | code_searcher cs; 32 | cs.load_index(index); 33 | 34 | for (auto it = cs.begin_files(); it != cs.end_files(); ++it) { 35 | if ((*it)->path == path) { 36 | dump_file(&cs, it->get()); 37 | return 0; 38 | } 39 | } 40 | 41 | fprintf(stderr, "No files matching path: %s\n", path.c_str()); 42 | 43 | return 0; 44 | } 45 | -------------------------------------------------------------------------------- /src/tools/grpc_server.h: -------------------------------------------------------------------------------- 1 | #ifndef CODESEARCH_GRPC_SERVER_H 2 | #define CODESEARCH_GRPC_SERVER_H 3 | 4 | #include "src/proto/livegrep.grpc.pb.h" 5 | #include 6 | #include 7 | 8 | class code_searcher; 9 | class tag_searcher; 10 | 11 | std::unique_ptr build_grpc_server(code_searcher *cs, 12 | code_searcher *tagdata, 13 | std::promise *reload_request); 14 | 15 | #endif /* CODESEARCH_GRPC_SERVER_H */ 16 | -------------------------------------------------------------------------------- /src/tools/limits.h: -------------------------------------------------------------------------------- 1 | #ifndef CODESEARCH_LIMITS_H 2 | #define CODESEARCH_LIMITS_H 3 | 4 | const int kMaxProgramSize = 4000; 5 | const int kMaxWidth = 200; 6 | 7 | #endif 8 | -------------------------------------------------------------------------------- /test/.gitignore: -------------------------------------------------------------------------------- 1 | codesearch_test 2 | -------------------------------------------------------------------------------- /test/BUILD: -------------------------------------------------------------------------------- 1 | config_setting( 2 | name = "darwin", 3 | values = {"host_cpu": "darwin"}, 4 | ) 5 | 6 | cc_test( 7 | name = "codesearch_test", 8 | size = "small", 9 | srcs = [ 10 | "codesearch_test.cc", 11 | "main.cc", 12 | "planner_test.cc", 13 | "tagsearch_test.cc", 14 | ], 15 | defines = select({ 16 | ":darwin": [ 17 | "GTEST_HAS_TR1_TUPLE=0", 18 | "GTEST_USE_OWN_TR1_TUPLE=1", 19 | ], 20 | "//conditions:default": [ 21 | "GTEST_HAS_TR1_TUPLE", 22 | "GTEST_USE_OWN_TR1_TUPLE=0", 23 | ], 24 | }), 25 | deps = [ 26 | "//src:codesearch", 27 | "//src/tools:grpc_server", 28 | "@com_google_googletest//:gtest", 29 | ], 30 | ) 31 | 32 | test_suite( 33 | name = "test", 34 | ) 35 | -------------------------------------------------------------------------------- /test/main.cc: -------------------------------------------------------------------------------- 1 | #include "gtest/gtest.h" 2 | 3 | int main(int ac, char* av[]) { 4 | testing::InitGoogleTest(&ac, av); 5 | return RUN_ALL_TESTS(); 6 | } 7 | -------------------------------------------------------------------------------- /test/tagsearch_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "gtest/gtest.h" 3 | 4 | #include "re2/re2.h" 5 | 6 | #include "src/tagsearch.h" 7 | #include "src/lib/debug.h" 8 | 9 | TEST(tagsearch_test, TagLinesFromQuery) { 10 | /* All variations on anchoring the beginning and ending of a string. */ 11 | 12 | query q = {}; 13 | q.line_pat.reset(new RE2("User")); 14 | string r = tag_searcher::create_tag_line_regex_from_query(&q); 15 | ASSERT_EQ(r, "^[^\t]*(User)[^\t]*\t"); 16 | 17 | q.line_pat.reset(new RE2("^User")); 18 | r = tag_searcher::create_tag_line_regex_from_query(&q); 19 | ASSERT_EQ(r, "^(User)[^\t]*\t"); 20 | 21 | q.line_pat.reset(new RE2("User$")); 22 | r = tag_searcher::create_tag_line_regex_from_query(&q); 23 | ASSERT_EQ(r, "^[^\t]*(User)\t"); 24 | 25 | q.line_pat.reset(new RE2("^User$")); 26 | r = tag_searcher::create_tag_line_regex_from_query(&q); 27 | ASSERT_EQ(r, "^(User)\t"); 28 | 29 | /* Briefer tests that each subsequent component is (a) correctly 30 | interpolated, and (b) in at least one case varies how it is 31 | anchored correctly. */ 32 | 33 | q.file_pat.reset(new RE2("models.py")); 34 | r = tag_searcher::create_tag_line_regex_from_query(&q); 35 | ASSERT_EQ(r, "^(User)\t[^\t]*(models.py)[^\t]*\t"); 36 | 37 | q.file_pat.reset(new RE2("^models.py")); 38 | r = tag_searcher::create_tag_line_regex_from_query(&q); 39 | ASSERT_EQ(r, "^(User)\t(models.py)[^\t]*\t"); 40 | 41 | q.tags_pat.reset(new RE2("c")); 42 | r = tag_searcher::create_tag_line_regex_from_query(&q); 43 | ASSERT_EQ(r, "^(User)\t(models.py)[^\t]*\t\\d+;\"\t.*(c).*$"); 44 | 45 | q.file_pat.reset(); 46 | r = tag_searcher::create_tag_line_regex_from_query(&q); 47 | ASSERT_EQ(r, "^(User)\t[^\t]+\t\\d+;\"\t.*(c).*$"); 48 | } 49 | -------------------------------------------------------------------------------- /third_party/BUILD: -------------------------------------------------------------------------------- 1 | cc_library( 2 | name = "utf8cpp", 3 | hdrs = [ 4 | "utf8cpp/source/utf8.h", 5 | "utf8cpp/source/utf8/checked.h", 6 | "utf8cpp/source/utf8/core.h", 7 | "utf8cpp/source/utf8/unchecked.h", 8 | ], 9 | includes = ["utf8cpp/source"], 10 | visibility = ["//visibility:public"], 11 | ) 12 | -------------------------------------------------------------------------------- /third_party/BUILD.divsufsort: -------------------------------------------------------------------------------- 1 | cc_library( 2 | name = "divsufsort", 3 | includes = [ 4 | "include/", 5 | ], 6 | hdrs = ["include/divsufsort.h"], 7 | srcs = glob(["lib/*.c", "include/*.h"], exclude=["include/divsufsort.h"]), 8 | visibility = [ "//visibility:public" ], 9 | copts = [ 10 | "-DHAVE_CONFIG_H" 11 | ], 12 | ) 13 | -------------------------------------------------------------------------------- /third_party/BUILD.gtest: -------------------------------------------------------------------------------- 1 | cc_library( 2 | name = "main", 3 | srcs = glob( 4 | ["src/*.cc", "src/*.h"], 5 | exclude = ["src/gtest-all.cc"] 6 | ), 7 | hdrs = glob(["include/**/*.h"]), 8 | copts = [ 9 | "-Iexternal/gtest", 10 | ], 11 | includes = [ 12 | "include", 13 | ], 14 | linkopts = ["-pthread"], 15 | visibility = ["//visibility:public"], 16 | ) 17 | -------------------------------------------------------------------------------- /third_party/BUILD.libgit2: -------------------------------------------------------------------------------- 1 | cc_library( 2 | name = "http_parser", 3 | srcs = ["deps/http-parser/http_parser.c"], 4 | hdrs = ["deps/http-parser/http_parser.h"], 5 | includes = ["deps/http-parser/"], 6 | ) 7 | 8 | cc_library( 9 | name = "headers", 10 | hdrs = glob([ 11 | "src/*.h", 12 | "include/git2/**/*.h", 13 | ]), 14 | includes = ["include"], 15 | ) 16 | 17 | cc_library( 18 | name = "zlib", 19 | srcs = glob(["deps/zlib/*.c"]), 20 | hdrs = glob(["deps/zlib/*.h"]), 21 | deps = [ 22 | ":headers", 23 | ], 24 | includes = [ 25 | "deps/zlib", 26 | ], 27 | copts = [ 28 | "-DSTDC=1", 29 | ], 30 | ) 31 | 32 | cc_library( 33 | name = "libgit2", 34 | hdrs = glob(["include/**/*.h"]), 35 | srcs = glob([ 36 | "src/*.c", "src/*.h", 37 | "src/hash/*.c", "src/hash/*.h", 38 | "src/unix/*.c", "src/unix/*.h", 39 | "src/transports/*.c", "src/transports/*.h", 40 | "src/xdiff/*.h", "src/xdiff/*.c", 41 | ], exclude = [ 42 | "src/hash/hash_win32.h", 43 | "src/hash/hash_win32.c", 44 | "src/transports/winhttp.c", 45 | ]) + [ 46 | 47 | ], 48 | deps = [ 49 | ":headers", 50 | ":http_parser", 51 | ":zlib", 52 | ], 53 | copts = [ 54 | "-Iexternal/com_github_libgit2/src", 55 | "-Wno-unused-function", 56 | ], 57 | visibility = ["//visibility:public"], 58 | ) 59 | -------------------------------------------------------------------------------- /third_party/sparseconfig.h: -------------------------------------------------------------------------------- 1 | /* 2 | * NOTE: This file is for internal use only. 3 | * Do not use these #defines in your own program! 4 | */ 5 | 6 | /* Namespace for Google classes */ 7 | #define GOOGLE_NAMESPACE ::google 8 | 9 | #ifdef __APPLE__ 10 | /* the location of the header defining hash functions */ 11 | #define HASH_FUN_H 12 | 13 | /* the namespace of the hash<> function */ 14 | #define HASH_NAMESPACE std 15 | #else 16 | /* the location of the header defining hash functions */ 17 | #define HASH_FUN_H 18 | 19 | /* the namespace of the hash<> function */ 20 | #define HASH_NAMESPACE std::tr1 21 | #endif 22 | 23 | /* Define to 1 if you have the header file. */ 24 | #define HAVE_INTTYPES_H 1 25 | 26 | /* Define to 1 if the system has the type `long long'. */ 27 | #define HAVE_LONG_LONG 1 28 | 29 | /* Define to 1 if you have the `memcpy' function. */ 30 | #define HAVE_MEMCPY 1 31 | 32 | /* Define to 1 if you have the header file. */ 33 | #define HAVE_STDINT_H 1 34 | 35 | /* Define to 1 if you have the header file. */ 36 | #define HAVE_SYS_TYPES_H 1 37 | 38 | /* Define to 1 if the system has the type `uint16_t'. */ 39 | #define HAVE_UINT16_T 1 40 | 41 | /* Define to 1 if the system has the type `u_int16_t'. */ 42 | #define HAVE_U_INT16_T 1 43 | 44 | /* Define to 1 if the system has the type `__uint16'. */ 45 | /* #undef HAVE___UINT16 */ 46 | 47 | /* The system-provided hash function including the namespace. */ 48 | #define SPARSEHASH_HASH HASH_NAMESPACE::hash 49 | 50 | /* Stops putting the code inside the Google namespace */ 51 | #define _END_GOOGLE_NAMESPACE_ } 52 | 53 | /* Puts following code inside the Google namespace */ 54 | #define _START_GOOGLE_NAMESPACE_ namespace google { 55 | -------------------------------------------------------------------------------- /third_party/utf8cpp/doc/ReleaseNotes: -------------------------------------------------------------------------------- 1 | utf8 cpp library 2 | Release 2.3.4 3 | 4 | A minor bug fix release. Thanks to all who reported bugs. 5 | 6 | Note: Version 2.3.3 contained a regression, and therefore was removed. 7 | 8 | Changes from version 2.3.2 9 | - Bug fix [39]: checked.h Line 273 and unchecked.h Line 182 have an extra ';' 10 | - Bug fix [36]: replace_invalid() only works with back_inserter 11 | 12 | Files included in the release: utf8.h, core.h, checked.h, unchecked.h, utf8cpp.html, ReleaseNotes 13 | -------------------------------------------------------------------------------- /third_party/utf8cpp/source/utf8.h: -------------------------------------------------------------------------------- 1 | // Copyright 2006 Nemanja Trifunovic 2 | 3 | /* 4 | Permission is hereby granted, free of charge, to any person or organization 5 | obtaining a copy of the software and accompanying documentation covered by 6 | this license (the "Software") to use, reproduce, display, distribute, 7 | execute, and transmit the Software, and to prepare derivative works of the 8 | Software, and to permit third-parties to whom the Software is furnished to 9 | do so, all subject to the following: 10 | 11 | The copyright notices in the Software and this entire statement, including 12 | the above license grant, this restriction and the following disclaimer, 13 | must be included in all copies of the Software, in whole or in part, and 14 | all derivative works of the Software, unless such copies or derivative 15 | works are solely in the form of machine-executable object code generated by 16 | a source language processor. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 21 | SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE 22 | FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 23 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 24 | DEALINGS IN THE SOFTWARE. 25 | */ 26 | 27 | 28 | #ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 29 | #define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 30 | 31 | #include "utf8/checked.h" 32 | #include "utf8/unchecked.h" 33 | 34 | #endif // header guard 35 | -------------------------------------------------------------------------------- /tools/build_defs/BUILD: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/livegrep/0a794275458b50a8da535a305435104e46d7c75e/tools/build_defs/BUILD -------------------------------------------------------------------------------- /tools/build_defs/go_externals.bzl: -------------------------------------------------------------------------------- 1 | load( 2 | "@bazel_gazelle//:deps.bzl", 3 | "go_repository", 4 | ) 5 | 6 | def _github(repo, commit): 7 | name = "com_github_" + repo.replace("/", "_").replace("-", "_").replace(".", "_") 8 | importpath = "github.com/" + repo 9 | return struct( 10 | name = name, 11 | commit = commit, 12 | importpath = importpath, 13 | ) 14 | 15 | def _golang_x(pkg, commit): 16 | name = "org_golang_x_" + pkg 17 | importpath = "golang.org/x/" + pkg 18 | return struct( 19 | name = name, 20 | commit = commit, 21 | importpath = importpath, 22 | ) 23 | 24 | _externals = [ 25 | _golang_x("net", "d212a1ef2de2f5d441c327b8f26cf3ea3ea9f265"), 26 | _golang_x("text", "a9a820217f98f7c8a207ec1e45a874e1fe12c478"), 27 | _golang_x("oauth2", "a6bd8cefa1811bd24b86f8902872e4e8225f74c4"), 28 | struct( 29 | name = "org_golang_google_appengine", 30 | commit = "170382fa85b10b94728989dfcf6cc818b335c952", 31 | importpath = "google.golang.org/appengine/", 32 | remote = "https://github.com/golang/appengine", 33 | vcs = "git", 34 | ), 35 | _github("google/go-github", "e8d46665e050742f457a58088b1e6b794b2ae966"), 36 | _github("honeycombio/libhoney-go", "a8716c5861ae19c1e2baaad52dd59ba64b902bde"), 37 | _github("nelhage/go.cli", "2aeb96ef8025f3646befae8353b90f95e9e79bdc"), 38 | _github("bmizerany/pat", "c068ca2f0aacee5ac3681d68e4d0a003b7d1fd2c"), 39 | _github("google/go-querystring", "53e6ce116135b80d037921a7fdd5138cf32d7a8a"), 40 | _github("facebookgo/muster", "fd3d7953fd52354a74b9f6b3d70d0c9650c4ec2a"), 41 | _github("facebookgo/limitgroup", "6abd8d71ec01451d7f1929eacaa263bbe2935d05"), 42 | _github("facebookgo/clock", "600d898af40aa09a7a93ecb9265d87b0504b6f03"), 43 | struct( 44 | name = "in_gopkg_alexcesaro_statsd_v2", 45 | commit = "7fea3f0d2fab1ad973e641e51dba45443a311a90", 46 | importpath = "gopkg.in/alexcesaro/statsd.v2", 47 | ), 48 | struct( 49 | name = "in_gopkg_check_v1", 50 | commit = "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec", 51 | importpath = "gopkg.in/check.v1", 52 | ), 53 | ] 54 | 55 | def go_externals(): 56 | for ext in _externals: 57 | if hasattr(ext, "vcs"): 58 | go_repository( 59 | name = ext.name, 60 | commit = ext.commit, 61 | importpath = ext.importpath, 62 | remote = ext.remote, 63 | vcs = ext.vcs, 64 | ) 65 | else: 66 | go_repository( 67 | name = ext.name, 68 | commit = ext.commit, 69 | importpath = ext.importpath, 70 | ) 71 | -------------------------------------------------------------------------------- /tools/build_defs/libgit2.bzl: -------------------------------------------------------------------------------- 1 | def _new_libgit2_archive_impl(ctx): 2 | tgz = "libgit2-" + ctx.attr.version + ".tar.gz" 3 | ctx.download( 4 | ctx.attr.url, 5 | tgz, 6 | ctx.attr.sha256, 7 | ) 8 | ctx.symlink(ctx.attr.build_file, "BUILD") 9 | cmd = ctx.execute([ 10 | "tar", "-xzf", tgz, 11 | "--strip-components=1", 12 | "--exclude=libgit2-" + ctx.attr.version + "/tests" 13 | ]) 14 | 15 | if cmd.return_code != 0: 16 | fail("error unpacking: " + cmd.stderr) 17 | 18 | new_libgit2_archive = repository_rule( 19 | implementation=_new_libgit2_archive_impl, 20 | attrs = { 21 | "url": attr.string(mandatory = True), 22 | "sha256": attr.string(), 23 | "build_file": attr.label(mandatory = True), 24 | "version": attr.string(mandatory = True), 25 | }) 26 | -------------------------------------------------------------------------------- /travisdeps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | mkdir -p deps 5 | 6 | if ! test -d deps/linux; then 7 | git clone --depth=1 --branch v3.17 https://github.com/torvalds/linux deps/linux 8 | fi 9 | -------------------------------------------------------------------------------- /web/3d/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | load("@org_dropbox_rules_node//node:defs.bzl", "node_internal_module") 4 | 5 | node_internal_module( 6 | name = "html", 7 | srcs = ["html.js"], 8 | main = "html.js", 9 | ) 10 | -------------------------------------------------------------------------------- /web/3d/html.js: -------------------------------------------------------------------------------- 1 | function HTMLFactory() {} 2 | (function() { 3 | "use strict"; 4 | /*** 5 | * A convenient way to create DOM elements. ('cls' will be 6 | * automatically expanded to 'class', since 'class' may not appear as 7 | * a key of an object, even in quotes, in Safari.) 8 | * 9 | * DIV({cls: "mydiv", style: "color: blue;"}, [ 10 | * "Some text", 11 | * A({href: "/some/location"}, ["A link"]), 12 | * DIV({cls: "emptydiv"}), 13 | * // if an object is inserted, the value of its 'element' 14 | * // attribute will be used 15 | * myView, 16 | * DIV([ 17 | * "Both the attributes and the contents are optional", 18 | * ["Lists", "are", "flattened"] 19 | * }) 20 | * ]); 21 | */ 22 | 23 | // XXX find a place to document the contract for *View classes -- they 24 | // should have an attribute named 'element' 25 | 26 | // XXX consider not requiring the contents to be wrapped in an 27 | // array. eg: DIV({stuff: 12}, "thing1", "thing2"). backwards 28 | // compatible with current behavior due to array flattening. could 29 | // eliminate spurious wrapper div inserted by Layout.TwoColumnsFixedRight 30 | 31 | // XXX allow style to be set as an object 32 | var event_names = { 33 | blur: true, 34 | change: true, 35 | click: true, 36 | dblclick: true, 37 | error: true, 38 | focus: true, 39 | focusin: true, 40 | focusout: true, 41 | keydown: true, 42 | keypress: true, 43 | keyup: true, 44 | load: true, 45 | mousedown: true, 46 | mouseenter: true, 47 | mouseleave: true, 48 | mousemove: true, 49 | mouseout: true, 50 | mouseover: true, 51 | mouseup: true, 52 | resize: true, 53 | scroll: true, 54 | select: true, 55 | submit: true 56 | }; 57 | 58 | // All HTML4 elements, excluding deprecated element 59 | // http://www.w3.org/TR/html4/index/elements.html 60 | // also excluding the following elements that seem unlikely to be used in the body: 61 | // HEAD, HTML, LINK, MAP, META, NOFRAMES, NOSCRIPT, STYLE, TITLE 62 | _(('A ABBR ACRONYM B BDO BIG BLOCKQUOTE BR BUTTON CAPTION CITE CODE COL ' + 63 | 'COLGROUP DD DEL DFN DIV DL DT EM FIELDSET FORM H1 H2 H3 H4 H5 H6 HR ' + 64 | 'I IFRAME IMG INPUT INS KBD LABEL LEGEND LI OBJECT OL OPTGROUP OPTION ' + 65 | 'P PARAM PRE Q S SAMP SCRIPT SELECT SMALL SPAN STRIKE STRONG SUB SUP TABLE ' + 66 | 'TBODY TD TEXTAREA TFOOT TH THEAD TR TT U UL VAR').split(' ')).forEach( 67 | function (tag) { 68 | HTMLFactory.prototype[tag.toLowerCase()] = function (arg1, arg2) { 69 | var attrs, contents; 70 | if (arg2) { 71 | attrs = arg1; 72 | contents = arg2; 73 | } else { 74 | if (arg1 instanceof Array) { 75 | attrs = {}; 76 | contents = arg1; 77 | } else { 78 | attrs = arg1; 79 | contents = []; 80 | } 81 | } 82 | var elt = document.createElement(tag); 83 | for (var a in attrs) { 84 | if (a === 'cls') 85 | elt.setAttribute('class', attrs[a]); 86 | else if (a === '_for') 87 | elt.setAttribute('for', attrs[a]); 88 | else if (event_names[a]) 89 | // XXX creates a dependency on jQuery.. ick.. 90 | ($(elt)[a])(attrs[a]); 91 | else 92 | elt.setAttribute(a, attrs[a]); 93 | } 94 | var addChildren = function (children) { 95 | _(children).forEach(function (c) { 96 | if (!c && c !== '') 97 | throw new Error("Bad value for element body: " + c); 98 | else if (c instanceof Array) 99 | addChildren(c); 100 | else if (typeof(c) === "string") 101 | elt.appendChild(document.createTextNode(c)); 102 | else if (typeof(c) === "number") 103 | elt.appendChild(document.createTextNode(c + "")); 104 | else if ('element' in c) 105 | addChildren([c.element]); 106 | else 107 | elt.appendChild(c); 108 | }); 109 | }; 110 | addChildren(contents); 111 | return elt; 112 | }; 113 | }); 114 | })(); 115 | 116 | module.exports = { 117 | HTMLFactory: HTMLFactory 118 | } 119 | -------------------------------------------------------------------------------- /web/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | load("@org_dropbox_rules_node//node:defs.bzl", "webpack_build") 4 | load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") 5 | 6 | filegroup( 7 | name = "templates", 8 | srcs = glob(["templates/**/*"]), 9 | ) 10 | 11 | filegroup( 12 | name = "htdocs", 13 | srcs = glob(["htdocs/**"]), 14 | ) 15 | 16 | webpack_build( 17 | name = "webpack_build", 18 | srcs = glob([ 19 | "src/**", 20 | ]), 21 | outs = [ 22 | "htdocs/assets/js/bundle.js", 23 | ], 24 | config = "webpack.config.js", 25 | deps = [ 26 | "//web/3d:html", 27 | "//web/npm/css-loader", 28 | "//web/npm/style-loader", 29 | "@org_dropbox_rules_node//node/dbx-bazel-utils", 30 | ], 31 | ) 32 | 33 | genrule( 34 | name = "asset_hashes", 35 | srcs = [ 36 | "//web:htdocs", 37 | "//web:webpack_build", 38 | ], 39 | outs = ["hashes.txt"], 40 | cmd = "sha256sum $(SRCS) | sed \"s_ $(BINDIR)/_ _\" | sed \"s_ web/htdocs/_ _\"> $@", 41 | ) 42 | 43 | pkg_tar( 44 | name = "assets", 45 | srcs = [ 46 | ":asset_hashes", 47 | ":htdocs", 48 | ":templates", 49 | ":webpack_build", 50 | ], 51 | package_dir = "web/", 52 | strip_prefix = ".", 53 | ) 54 | -------------------------------------------------------------------------------- /web/htdocs/assets/3d/README.bootstrap: -------------------------------------------------------------------------------- 1 | bootstrap.min.css was generated with the boostrap customizer. We commit the 2 | customized CSS, but still load the (standard) JS and fonts from cdnjs. Note 3 | that the CSS is customized to refer explicitly to the fonts hosted on cdnjs. 4 | 5 | https://getbootstrap.com/docs/3.3/customize/?id=2af5b28271904f20b716d93bf37253e0 6 | https://gist.github.com/anonymous/2af5b28271904f20b716d93bf37253e0 7 | -------------------------------------------------------------------------------- /web/htdocs/assets/css/blame.css: -------------------------------------------------------------------------------- 1 | body { 2 | position: relative; /* so #hashes position is based on 's */ 3 | display: inline-block; /* allow body to expand to fit wide files */ 4 | white-space: pre; 5 | font-family: "Menlo", "Consolas", "Monaco", monospace; 6 | font-size: 12px; 7 | } 8 | 9 | /* Keep the header at the top. */ 10 | #header { 11 | position: sticky; 12 | top: 0px; 13 | background-color: white; 14 | } 15 | 16 | /* Put the content div, that we deliver first, on the right; then 17 | position the blame div to its left. */ 18 | .rtl { 19 | direction: rtl; /* so we can deliver content first, then blame */ 20 | text-align: left; /* but keep against left margin */ 21 | } 22 | .rtl > * { 23 | vertical-align: top; 24 | display: inline-block; 25 | direction: ltr; /* reset back to normal */ 26 | } 27 | 28 | /* Leave the hash links mostly undecorated, and gray unless the user 29 | mouses over them and triggers our js to add the "highlight" class: */ 30 | 31 | #hashes > a { 32 | text-decoration: none; 33 | } 34 | #hashes > a[href] { /* leave hash links gray until mouseover*/ 35 | color: gray; 36 | } 37 | #hashes > a.highlight { /* turn hash links black when moused over */ 38 | color: black; 39 | } 40 | 41 | /* We have several uses for a box extending right from a #hashes anchor 42 | to highlight a line of code with a background color; here is the CSS 43 | that all such boxes have in common (and which has no effect until the 44 | non-existent ":before" is given a "content"): */ 45 | 46 | #hashes > a:before { 47 | position: absolute; 48 | left: 0; 49 | z-index: -1; /* place behind text of file */ 50 | box-sizing: border-box; /* so border-width doesn't oversize it */ 51 | width: 100%; 52 | border-width: 0 0 0 45ch; 53 | border-style: solid; 54 | border-color: white; 55 | } 56 | body.wide #hashes > a:before { 57 | border-width: 0 0 0 49ch; 58 | } 59 | 60 | /* When a commit hash is being highlighted, paint a green or red 61 | background behind all of the lines that it adds or deletes: */ 62 | 63 | #hashes > a.highlight:before { 64 | background-color: #c0f8c0; /* light green highlight */ 65 | color: green; 66 | content: "+"; 67 | } 68 | #hashes > a.highlight.d:before { 69 | background-color: #f8c0c0; /* light red highlight */ 70 | color: red; 71 | content: "-"; 72 | } 73 | 74 | /* Make the line that the mouse is hovering over stand out a little by 75 | giving it a slightly darker background: */ 76 | 77 | #hashes > a.highlight:hover:before { 78 | background-color: #90EE90; /* darker green highlight */ 79 | } 80 | #hashes > a.highlight.d:hover:before { 81 | background-color: #EE9090; /* darker red highlight */ 82 | } 83 | 84 | /* When a diff is being viewed, place permanent red and green background 85 | colors behind the lines it adds and deletes: */ 86 | 87 | #hashes > b:before { 88 | position: absolute; 89 | left: 0; 90 | box-sizing: border-box; 91 | width: 100%; 92 | border-width: 0 0 0 50ch; 93 | border-style: solid; 94 | border-color: white; 95 | z-index: -2; /* place behind text of file and highlights */ 96 | content: " "; 97 | } 98 | #hashes > b.a:before { 99 | background-color: #c0f8c0; /* light green highlight */ 100 | } 101 | #hashes > b.d:before { 102 | background-color: #f8c0c0; /* light red highlight */ 103 | } 104 | 105 | /* Highlight the target line that is named in the URL's fragment ID. */ 106 | 107 | #hashes > a:target:after { 108 | z-index: -1; /* place behind text of file */ 109 | position: absolute; 110 | left: 0; 111 | box-sizing: border-box; 112 | width: 100%; 113 | margin-top: -2px; 114 | border: 2px solid black; 115 | content: " "; 116 | } 117 | -------------------------------------------------------------------------------- /web/htdocs/assets/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/livegrep/0a794275458b50a8da535a305435104e46d7c75e/web/htdocs/assets/img/favicon.ico -------------------------------------------------------------------------------- /web/htdocs/assets/raw-js/blame.js: -------------------------------------------------------------------------------- 1 | (function f() { 2 | var $body = $('body'); 3 | 4 | /* In the file view, highlight the contents of each diff whose 5 | commit the user mouses over. */ 6 | 7 | if ($body.hasClass('blamefile')) { 8 | $body.on('mouseenter', '#hashes > a', function(e) { 9 | var href = $(e.target).attr('href') || ""; 10 | var i = href.indexOf('.'); 11 | if (i == -1) return; 12 | var commitHash = href.substring(0, i); 13 | var cls = 'highlight ' + href.substr(i + 1, 1); 14 | $('#hashes a[href^="' + commitHash + '"]').addClass(cls); 15 | }); 16 | $body.on('mouseleave', '#hashes > a', function(e) { 17 | var href = $(e.target).attr('href') || ""; 18 | var i = href.indexOf('.'); 19 | if (i == -1) return; 20 | var commitHash = href.substring(0, i); 21 | var cls = 'highlight ' + href.substr(i + 1, 1); 22 | $('#hashes a[href^="' + commitHash + '"]').removeClass(cls); 23 | }); 24 | } 25 | 26 | /* When the user clicks a hash, remember the line's y coordinate, 27 | and warp it back to its current location when we land. */ 28 | 29 | $body.on('click', '#hashes > a', function(e) { 30 | var y = $(e.currentTarget).offset().top - $(window).scrollTop(); 31 | Cookies.set("target_y", y, {expires: 1}); 32 | // (Then, let the click proceed with its usual effect.) 33 | }); 34 | 35 | var previous_y = Cookies.get("target_y"); 36 | if (typeof previous_y !== "undefined") { 37 | Cookies.remove("target_y"); 38 | } 39 | 40 | $(document).ready(function() { 41 | var target = $(":target"); 42 | if (target.length === 0) 43 | return; 44 | var y = previous_y; 45 | if (typeof y === "undefined") 46 | y = window.innerHeight / 2; // default to middle of viewport 47 | scroll_y = target.offset().top - y; 48 | scroll_y = Math.max(0, scroll_y); // clip to lower bound of 0 49 | window.scrollTo(0, scroll_y); 50 | }); 51 | })(); 52 | -------------------------------------------------------------------------------- /web/npm/style-loader/npm-shrinkwrap.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "npm-gen-style-loader", 3 | "version": "0.18.2", 4 | "dependencies": { 5 | "style-loader": { 6 | "version": "0.18.2", 7 | "from": "style-loader@0.18.2", 8 | "resolved": "https://registry.npmjs.org/style-loader/-/style-loader-0.18.2.tgz", 9 | "dependencies": { 10 | "ajv": { 11 | "version": "5.2.2", 12 | "from": "ajv@>=5.0.0 <6.0.0", 13 | "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.2.2.tgz" 14 | }, 15 | "big.js": { 16 | "version": "3.1.3", 17 | "from": "big.js@>=3.1.3 <4.0.0", 18 | "resolved": "https://registry.npmjs.org/big.js/-/big.js-3.1.3.tgz" 19 | }, 20 | "co": { 21 | "version": "4.6.0", 22 | "from": "co@>=4.6.0 <5.0.0", 23 | "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz" 24 | }, 25 | "emojis-list": { 26 | "version": "2.1.0", 27 | "from": "emojis-list@>=2.0.0 <3.0.0", 28 | "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-2.1.0.tgz" 29 | }, 30 | "fast-deep-equal": { 31 | "version": "1.0.0", 32 | "from": "fast-deep-equal@>=1.0.0 <2.0.0", 33 | "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz" 34 | }, 35 | "json-schema-traverse": { 36 | "version": "0.3.1", 37 | "from": "json-schema-traverse@>=0.3.0 <0.4.0", 38 | "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz" 39 | }, 40 | "json-stable-stringify": { 41 | "version": "1.0.1", 42 | "from": "json-stable-stringify@>=1.0.1 <2.0.0", 43 | "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz" 44 | }, 45 | "json5": { 46 | "version": "0.5.1", 47 | "from": "json5@>=0.5.0 <0.6.0", 48 | "resolved": "https://registry.npmjs.org/json5/-/json5-0.5.1.tgz" 49 | }, 50 | "jsonify": { 51 | "version": "0.0.0", 52 | "from": "jsonify@>=0.0.0 <0.1.0", 53 | "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz" 54 | }, 55 | "loader-utils": { 56 | "version": "1.1.0", 57 | "from": "loader-utils@>=1.0.2 <2.0.0", 58 | "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.1.0.tgz" 59 | }, 60 | "schema-utils": { 61 | "version": "0.3.0", 62 | "from": "schema-utils@>=0.3.0 <0.4.0", 63 | "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-0.3.0.tgz" 64 | } 65 | } 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /web/src/codesearch/codesearch.js: -------------------------------------------------------------------------------- 1 | $ = require('jquery'); 2 | _ = require('underscore'); 3 | 4 | "use strict"; 5 | var Codesearch = function() { 6 | return { 7 | delegate: null, 8 | retry_time: 50, 9 | next_search: null, 10 | in_flight: null, 11 | connect: function(delegate) { 12 | if (delegate !== undefined) 13 | Codesearch.delegate = delegate; 14 | if (Codesearch.delegate.on_connect) 15 | setTimeout(Codesearch.delegate.on_connect, 0) 16 | }, 17 | new_search: function(opts) { 18 | Codesearch.next_search = opts; 19 | if (Codesearch.in_flight == null) 20 | Codesearch.dispatch() 21 | }, 22 | dispatch: function() { 23 | if (!Codesearch.next_search) 24 | return; 25 | Codesearch.in_flight = Codesearch.next_search; 26 | Codesearch.next_search = null; 27 | 28 | var opts = Codesearch.in_flight; 29 | 30 | var url = "/api/v1/search/"; 31 | if ('backend' in opts) { 32 | url = url + opts.backend; 33 | } 34 | var q = { 35 | q: opts.q, 36 | fold_case: opts.fold_case, 37 | regex: opts.regex, 38 | repo: opts.repo 39 | }; 40 | 41 | url = url + "?" + $.param(q); 42 | 43 | var xhr = $.getJSON(url); 44 | var start = new Date(); 45 | xhr.done(function (data) { 46 | var elapsed = new Date() - start; 47 | data.results.forEach(function (r) { 48 | Codesearch.delegate.match(opts.id, r); 49 | }); 50 | data.file_results.forEach(function (r) { 51 | Codesearch.delegate.file_match(opts.id, r); 52 | }); 53 | Codesearch.delegate.search_done(opts.id, elapsed, data.search_type, data.info.why); 54 | }); 55 | xhr.fail(function(data) { 56 | window._err = data; 57 | if (data.status >= 400 && data.status < 500) { 58 | var err = JSON.parse(data.responseText); 59 | Codesearch.delegate.error(opts.id, err.error.message); 60 | } else { 61 | var message = "Cannot connect to server"; 62 | if (data.status) { 63 | message = "Bad response " + data.status + " from server"; 64 | } 65 | Codesearch.delegate.error(opts.id, message); 66 | console.log("server error", data.status, data.responseText); 67 | } 68 | }); 69 | xhr.always(function() { 70 | Codesearch.in_flight = null; 71 | setTimeout(Codesearch.dispatch, 0); 72 | }); 73 | } 74 | }; 75 | }(); 76 | 77 | module.exports = { 78 | Codesearch: Codesearch 79 | } 80 | -------------------------------------------------------------------------------- /web/src/codesearch/repo_selector.js: -------------------------------------------------------------------------------- 1 | var $ = require('jquery'); 2 | var _ = require('underscore'); 3 | 4 | function init() { 5 | $('#repos').selectpicker({ 6 | actionsBox: true, 7 | selectedTextFormat: 'count > 4', 8 | countSelectedText: '({0} repositories)', 9 | noneSelectedText: '(all repositories)', 10 | liveSearch: true, 11 | width: '20em' 12 | }); 13 | $(window).on('keyup', '.bootstrap-select .bs-searchbox input', function(event) { 14 | var keycode = (event.keyCode ? event.keyCode : event.which); 15 | if(keycode == '13'){ 16 | $(this).val(""); 17 | $("#repos").selectpicker('refresh'); 18 | } 19 | }); 20 | $(window).keyup(function (keyevent) { 21 | var code = (keyevent.keyCode ? keyevent.keyCode : keyevent.which); 22 | if (code == 9 && $('.bootstrap-select button:focus').length) { 23 | $("#repos").selectpicker('toggle'); 24 | $('.bootstrap-select .bs-searchbox input').focus(); 25 | } 26 | }); 27 | } 28 | 29 | function updateOptions(newOptions) { 30 | // Skip update if the options are the same, to avoid losing selected state. 31 | var currentOptions = []; 32 | $('#repos').find('option').each(function(){ 33 | currentOptions.push($(this).attr('value')); 34 | }); 35 | if (_.isEqual(currentOptions, newOptions)) { 36 | return; 37 | } 38 | 39 | $('#repos').empty(); 40 | for (var i = 0; i < newOptions.length; i++) { 41 | var option = newOptions[i]; 42 | $('#repos').append($('