├── .gitbook.yaml ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── CODEOWNERS ├── CONTRIBUTING.md ├── DCO ├── Dockerfile ├── Jenkinsfile ├── LICENSE ├── MAINTAINERS ├── Makefile ├── README.md ├── _testdata ├── 05893125684f2d3943cd84a7ab2b75e53668fba1.siva ├── fff7062de8474d10a67d417ccea87ba6f58ca81d.siva ├── fff840f8784ef162dc83a1465fc5763d890b68ba.siva ├── library.yaml ├── not-permission │ └── .gitkeep ├── not-siva.txt ├── regression.yml ├── repositories-link └── repositories │ └── repositories-link ├── blobs.go ├── blobs_test.go ├── checksum.go ├── checksum_test.go ├── cmd └── gitbase │ ├── command │ ├── server.go │ ├── server_test.go │ └── version.go │ └── main.go ├── commit_blobs.go ├── commit_blobs_test.go ├── commit_files.go ├── commit_files_test.go ├── commit_trees.go ├── commit_trees_test.go ├── commits.go ├── commits_test.go ├── common_test.go ├── database.go ├── database_test.go ├── docs ├── README.md ├── assets │ ├── gitbase-schema.png │ └── gitbase_model.mwb ├── join-the-community.md └── using-gitbase │ ├── configuration.md │ ├── examples.md │ ├── functions.md │ ├── getting-started.md │ ├── indexes.md │ ├── optimize-queries.md │ ├── schema.md │ ├── security.md │ ├── supported-clients.md │ ├── supported-languages.md │ └── supported-syntax.md ├── e2e └── e2e_test.go ├── env.go ├── files.go ├── files_test.go ├── filters.go ├── filters_test.go ├── fs_error_test.go ├── go.mod ├── go.sum ├── index.go ├── index_test.go ├── init.sh ├── integration_test.go ├── internal ├── commitstats │ ├── commit.go │ ├── commit_test.go │ ├── common.go │ ├── file.go │ └── file_test.go ├── function │ ├── blame.go │ ├── blame_test.go │ ├── commit_file_stats.go │ ├── commit_file_stats_test.go │ ├── commit_stats.go │ ├── commit_stats_test.go │ ├── is_remote.go │ ├── is_remote_test.go │ ├── is_tag.go │ ├── is_tag_test.go │ ├── is_vendor.go │ ├── is_vendor_test.go │ ├── language.go │ ├── language_test.go │ ├── loc.go │ ├── loc_test.go │ ├── registry.go │ ├── uast.go │ ├── uast_test.go │ └── uast_utils.go └── rule │ ├── squashjoins.go │ └── squashjoins_test.go ├── packfiles.go ├── packfiles_test.go ├── partition.go ├── path_utils.go ├── path_utils_test.go ├── pull_request_template.md ├── ref_commits.go ├── ref_commits_test.go ├── references.go ├── references_test.go ├── regression_test.go ├── remotes.go ├── remotes_test.go ├── repositories.go ├── repositories_test.go ├── repository_pool.go ├── repository_pool_test.go ├── session.go ├── session_test.go ├── squash.go ├── squash_iterator.go ├── squash_iterator_test.go ├── table.go ├── table_test.go ├── tools └── rev-upgrade │ └── main.go ├── tree_entries.go ├── tree_entries_test.go └── utils.go /.gitbook.yaml: -------------------------------------------------------------------------------- 1 | structure: 2 | readme: README.md 3 | summary: docs/README.md -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | vendor 6 | 7 | # Folders 8 | _obj 9 | _test 10 | 11 | # Architecture specific extensions/prefixes 12 | *.[568vq] 13 | [568vq].out 14 | 15 | *.cgo1.go 16 | *.cgo2.c 17 | _cgo_defun.c 18 | _cgo_gotypes.go 19 | _cgo_export.* 20 | 21 | _testmain.go 22 | 23 | main 24 | gitbase 25 | *.exe 26 | *.test 27 | *.prof 28 | 29 | # CI 30 | .ci/ 31 | build/ 32 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go_import_path: github.com/src-d/gitbase 4 | go: 1.12.x 5 | 6 | env: 7 | - GO111MODULE=on GOPROXY=https://proxy.golang.org 8 | 9 | matrix: 10 | fast_finish: true 11 | 12 | before_script: 13 | - docker run -d --name bblfshd --privileged -p 9432:9432 bblfsh/bblfshd:v2.14.0-drivers 14 | - docker exec -it bblfshd bblfshctl driver list 15 | 16 | script: 17 | - make test-coverage codecov 18 | - make ci-e2e 19 | 20 | jobs: 21 | include: 22 | - os: linux 23 | sudo: required 24 | dist: trusty 25 | services: [docker] 26 | 27 | before_deploy: 28 | - make docker-push-latest-release 29 | - make static-package 30 | 31 | deploy: 32 | provider: releases 33 | api_key: $GITHUB_TOKEN 34 | file_glob: true 35 | file: build/*linux_amd64.tar.gz 36 | skip_cleanup: true 37 | on: 38 | tags: true 39 | 40 | - os: osx 41 | osx_image: xcode9.3 42 | 43 | before_install: 44 | - echo "skipping before_install for macOS" 45 | 46 | before_script: 47 | - echo "skipping before_script for macOS" 48 | 49 | script: 50 | - make packages || echo "" # will fail because of docker being missing 51 | - if [ ! -f "build/gitbase_darwin_amd64/gitbase" ]; then echo "gitbase binary not generated" && exit 1; fi 52 | - cd build 53 | - tar -cvzf "gitbase_${TRAVIS_TAG}_darwin_amd64.tar.gz" gitbase_darwin_amd64 54 | - cd .. 55 | 56 | deploy: 57 | provider: releases 58 | api_key: $GITHUB_TOKEN 59 | file_glob: true 60 | file: build/*darwin_amd64.tar.gz 61 | skip_cleanup: true 62 | on: 63 | tags: true 64 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @src-d/data-processing 2 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # source{d} Contributing Guidelines 2 | 3 | source{d} projects accept contributions via GitHub pull requests. 4 | This document outlines some of the 5 | conventions on development workflow, commit message formatting, contact points, 6 | and other resources to make it easier to get your contribution accepted. 7 | 8 | ## Certificate of Origin 9 | 10 | By contributing to this project you agree to the [Developer Certificate of 11 | Origin (DCO)](DCO). This document was created by the Linux Kernel community and is a 12 | simple statement that you, as a contributor, have the legal right to make the 13 | contribution. 14 | 15 | In order to show your agreement with the DCO you should include at the end of commit message, 16 | the following line: `Signed-off-by: John Doe `, using your real name. 17 | 18 | This can be done easily using the [`-s`](https://github.com/git/git/blob/b2c150d3aa82f6583b9aadfecc5f8fa1c74aca09/Documentation/git-commit.txt#L154-L161) flag on the `git commit`. 19 | 20 | ## Support Channels 21 | 22 | The official support channels, for both users and contributors, are: 23 | 24 | - GitHub issues: each repository has its own list of issues. 25 | - Slack: join the [source{d} Slack](https://join.slack.com/t/sourced-community/shared_invite/enQtMjc4Njk5MzEyNzM2LTFjNzY4NjEwZGEwMzRiNTM4MzRlMzQ4MmIzZjkwZmZlM2NjODUxZmJjNDI1OTcxNDAyMmZlNmFjODZlNTg0YWM) community. 26 | 27 | *Before opening a new issue or submitting a new pull request, it's helpful to 28 | search the project - it's likely that another user has already reported the 29 | issue you're facing, or it's a known issue that we're already aware of. 30 | 31 | ## How to Contribute 32 | 33 | Pull Requests (PRs) are the main and exclusive way to contribute code to source{d} projects. 34 | In order for a PR to be accepted it needs to pass a list of requirements: 35 | 36 | - The contribution must be correctly explained with natural language and providing a minimum working example that reproduces it. 37 | - All PRs must be written idiomatically: 38 | - for Go: formatted according to [gofmt](https://golang.org/cmd/gofmt/), and without any warnings from [go lint](https://github.com/golang/lint) nor [go vet](https://golang.org/cmd/vet/) 39 | - for other languages, similar constraints apply. 40 | - They should in general include tests, and those shall pass. 41 | - If the PR is a bug fix, it has to include a new unit test that fails before the patch is merged. 42 | - If the PR is a new feature, it has to come with a suite of unit tests, that tests the new functionality. 43 | - In any case, all the PRs have to pass the personal evaluation of at least one of the [maintainers](MAINTAINERS). 44 | 45 | ### Format of the commit message 46 | 47 | Every commit message should describe what was changed, under which context and, if applicable, the GitHub issue it relates to: 48 | 49 | ``` 50 | plumbing: packp, Skip argument validations for unknown capabilities. Fixes #623 51 | ``` 52 | 53 | The format can be described more formally as follows: 54 | 55 | ``` 56 | : , . [Fixes #] 57 | ``` 58 | -------------------------------------------------------------------------------- /DCO: -------------------------------------------------------------------------------- 1 | Developer Certificate of Origin 2 | Version 1.1 3 | 4 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 5 | 660 York Street, Suite 102, 6 | San Francisco, CA 94110 USA 7 | 8 | Everyone is permitted to copy and distribute verbatim copies of this 9 | license document, but changing it is not allowed. 10 | 11 | 12 | Developer's Certificate of Origin 1.1 13 | 14 | By making a contribution to this project, I certify that: 15 | 16 | (a) The contribution was created in whole or in part by me and I 17 | have the right to submit it under the open source license 18 | indicated in the file; or 19 | 20 | (b) The contribution is based upon previous work that, to the best 21 | of my knowledge, is covered under an appropriate open source 22 | license and I have the right under that license to submit that 23 | work with modifications, whether created in whole or in part 24 | by me, under the same open source license (unless I am 25 | permitted to submit under a different license), as indicated 26 | in the file; or 27 | 28 | (c) The contribution was provided directly to me by some other 29 | person who certified (a), (b) or (c) and I have not modified 30 | it. 31 | 32 | (d) I understand and agree that this project and the contribution 33 | are public and that a record of the contribution (including all 34 | personal information I submit with it, including my sign-off) is 35 | maintained indefinitely and may be redistributed consistent with 36 | this project or the open source license(s) involved. 37 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | #================================ 2 | # Stage 1: Build Gitbase 3 | #================================ 4 | FROM golang:1.11-alpine as builder 5 | 6 | ENV GITBASE_REPO=github.com/src-d/gitbase 7 | ENV GITBASE_PATH=$GOPATH/src/$GITBASE_REPO 8 | 9 | RUN apk add --update --no-cache libxml2-dev git make bash gcc g++ curl oniguruma-dev oniguruma 10 | 11 | COPY . $GITBASE_PATH 12 | WORKDIR $GITBASE_PATH 13 | 14 | ENV GO_BUILD_ARGS="-o /bin/gitbase" 15 | ENV GO_BUILD_PATH="./cmd/gitbase" 16 | ENV GO111MODULE=on 17 | ENV GOPROXY=https://proxy.golang.org 18 | 19 | RUN make static-build 20 | 21 | #================================= 22 | # Stage 2: Start Gitbase Server 23 | #================================= 24 | FROM alpine:3.8 25 | 26 | RUN apk add --no-cache mysql-client 27 | 28 | RUN mkdir -p /opt/repos 29 | 30 | EXPOSE 3306 31 | 32 | ENV TINI_VERSION v0.18.0 33 | ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static-amd64 /tini 34 | RUN chmod +x /tini 35 | ENTRYPOINT ["/tini", "--"] 36 | 37 | ENV GITBASE_USER=root 38 | ENV GITBASE_PASSWORD="" 39 | ENV GITBASE_REPOS=/opt/repos 40 | ENV MYSQL_HOST=127.0.0.1 41 | 42 | # copy build artifacts 43 | COPY --from=builder /bin/gitbase /bin/gitbase 44 | ADD init.sh ./init.sh 45 | RUN chmod +x ./init.sh 46 | 47 | ENTRYPOINT ["./init.sh"] 48 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent { 3 | kubernetes { 4 | label 'regression-gitbase' 5 | inheritFrom 'performance' 6 | defaultContainer 'regression-gitbase' 7 | nodeSelector 'srcd.host/type=jenkins-worker' 8 | containerTemplate { 9 | name 'regression-gitbase' 10 | image 'srcd/regression-gitbase:v0.3.4' 11 | ttyEnabled true 12 | command 'cat' 13 | } 14 | } 15 | } 16 | environment { 17 | GOPATH = "/go" 18 | GO_IMPORT_PATH = "github.com/src-d/regression-gibase" 19 | GO_IMPORT_FULL_PATH = "${env.GOPATH}/src/${env.GO_IMPORT_PATH}" 20 | GO111MODULE = "on" 21 | PROM_ADDRESS = "http://prom-pushgateway-prometheus-pushgateway.monitoring.svc.cluster.local:9091" 22 | PROM_JOB = "gitbase_perfomance" 23 | } 24 | triggers { pollSCM('0 0,12 * * *') } 25 | stages { 26 | stage('Run performance tests') { 27 | when { branch 'master' } 28 | steps { 29 | sh '/bin/regression --complexity=2 --csv --prom local:HEAD' 30 | } 31 | } 32 | stage('PR-run') { 33 | when { changeRequest target: 'master' } 34 | steps { 35 | sh '/bin/regression --complexity=0 remote:master local:HEAD' 36 | } 37 | } 38 | stage('Plot') { 39 | when { branch 'master' } 40 | steps { 41 | script { 42 | plotFiles = findFiles(glob: "plot_*.csv") 43 | plotFiles.each { 44 | echo "plot ${it.getName()}" 45 | sh "cat ${it.getName()}" 46 | plot( 47 | group: 'performance', 48 | csvFileName: it.getName(), 49 | title: it.getName(), 50 | numBuilds: '100', 51 | style: 'line', 52 | csvSeries: [[ 53 | displayTableFlag: false, 54 | exclusionValues: '', 55 | file: it.getName(), 56 | inclusionFlag: 'OFF', 57 | ]] 58 | ) 59 | } 60 | } 61 | } 62 | } 63 | stage('Run bblfsh mockup tests') { 64 | when { branch 'master' } 65 | steps { 66 | sh '/bin/regression-bblfsh-mockups local:HEAD' 67 | } 68 | } 69 | } 70 | post { 71 | success { 72 | slackSend (color: '#2eb886', message: "SUCCESS: `${env.JOB_NAME}` <${env.BUILD_URL}|build #${env.BUILD_NUMBER}>") 73 | } 74 | failure { 75 | slackSend (color: '#b82e60', message: "FAILED: `${env.JOB_NAME}` <${env.BUILD_URL}|build #${env.BUILD_NUMBER}>") 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /MAINTAINERS: -------------------------------------------------------------------------------- 1 | Miguel Molina (@erizocosmico) 2 | Antonio Navarro Perez (@ajnavarro) 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Package configuration 2 | PROJECT = gitbase 3 | COMMANDS = cmd/gitbase 4 | 5 | # Including ci Makefile 6 | CI_REPOSITORY ?= https://github.com/src-d/ci.git 7 | CI_PATH ?= $(shell pwd)/.ci 8 | CI_VERSION ?= v1 9 | 10 | UPGRADE_PRJ ?= "github.com/src-d/go-mysql-server" 11 | UPGRADE_REV ?= $(shell curl --silent "https://api.github.com/repos/src-d/go-mysql-server/commits/master" -H'Accept: application/vnd.github.VERSION.sha') 12 | 13 | MAKEFILE := $(CI_PATH)/Makefile.main 14 | $(MAKEFILE): 15 | git clone --quiet --branch $(CI_VERSION) --depth 1 $(CI_REPOSITORY) $(CI_PATH); 16 | 17 | -include $(MAKEFILE) 18 | 19 | 20 | upgrade: 21 | go run tools/rev-upgrade/main.go -p $(UPGRADE_PRJ) -r $(UPGRADE_REV) 22 | 23 | static-package: 24 | PACKAGE_NAME=gitbase_$(VERSION)_static_linux_amd64 ; \ 25 | docker rm gitbase-temp ; \ 26 | docker create --rm --name gitbase-temp $(DOCKER_ORG)/gitbase:$(VERSION) && \ 27 | mkdir -p build/$${PACKAGE_NAME} && \ 28 | docker cp gitbase-temp:/bin/gitbase build/$${PACKAGE_NAME} && \ 29 | cd build && \ 30 | tar czvf $${PACKAGE_NAME}.tar.gz $${PACKAGE_NAME} && \ 31 | docker rm gitbase-temp 32 | 33 | # target used in the Dockerfile to build the static binary 34 | static-build: VERSION = $(shell git describe --exact-match --tags 2>/dev/null || "dev-$(git rev-parse --short HEAD)$(test -n "`git status --porcelain`" && echo "-dirty" || true)") 35 | static-build: LD_FLAGS += -linkmode external -extldflags '-static -lz' -s -w 36 | static-build: GO_BUILD_PATH ?= github.com/src-d/gitbase/... 37 | static-build: 38 | go build $(GO_BUILD_ARGS) -ldflags="$(LD_FLAGS)" -v $(GO_BUILD_PATH) 39 | 40 | ci-e2e: packages 41 | go test ./e2e -gitbase-version="$(TRAVIS_TAG)" \ 42 | -must-run \ 43 | -gitbase-bin="$(TRAVIS_BUILD_DIR)/build/gitbase_linux_amd64/gitbase" \ 44 | -gitbase-repos="$(TRAVIS_BUILD_DIR)/.." -v 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gitbase [![GitHub version](https://badge.fury.io/gh/src-d%2Fgitbase.svg)](https://github.com/src-d/gitbase/releases) [![Build Status](https://travis-ci.com/src-d/gitbase.svg?branch=master)](https://travis-ci.com/src-d/gitbase) [![codecov](https://codecov.io/gh/src-d/gitbase/branch/master/graph/badge.svg)](https://codecov.io/gh/src-d/gitbase) [![GoDoc](https://godoc.org/gopkg.in/src-d/gitbase.v0?status.svg)](https://godoc.org/gopkg.in/src-d/gitbase.v0) [![Go Report Card](https://goreportcard.com/badge/github.com/src-d/gitbase)](https://goreportcard.com/report/github.com/src-d/gitbase) 2 | 3 | **gitbase**, is a SQL database interface to Git repositories. 4 | 5 | This project is now part of [source{d} Community Edition](https://sourced.tech/products/community-edition/), 6 | which provides the simplest way to get started with a single command. 7 | Visit [https://docs.sourced.tech/community-edition](https://docs.sourced.tech/community-edition) for more information. 8 | 9 | It can be used to perform SQL queries about the Git history and 10 | about the [Universal AST](https://doc.bblf.sh/) of the code itself. gitbase is being built to work on top of any number of git repositories. 11 | 12 | gitbase implements the *MySQL* wire protocol, it can be accessed using any MySQL 13 | client or library from any language. 14 | 15 | [src-d/go-mysql-server](https://github.com/src-d/go-mysql-server) is the SQL engine implementation used by `gitbase`. 16 | 17 | ## Status 18 | 19 | The project is currently in **alpha** stage, meaning it's still lacking performance in a number of cases but we are working hard on getting a performant system able to process thousands of repositories in a single node. Stay tuned! 20 | 21 | ## Examples 22 | 23 | You can see some [query examples](/docs/using-gitbase/examples.md) in [gitbase documentation](/docs). 24 | 25 | ## Motivation and scope 26 | 27 | gitbase was born to ease the analysis of git repositories and their source code. 28 | 29 | Also, making it MySQL compatible, we provide the maximum compatibility between languages and existing tools. 30 | 31 | It comes as a single self-contained binary and it can be used as a standalone service. The service is able to process local repositories and integrates with existing tools and frameworks to simplify source code analysis on a large scale. 32 | The integration with Apache Spark is planned and is currently under active development. 33 | 34 | ## Further reading 35 | 36 | From here, you can directly go to [getting started](/docs/using-gitbase/getting-started.md). 37 | 38 | ## License 39 | 40 | Apache License Version 2.0, see [LICENSE](LICENSE) 41 | -------------------------------------------------------------------------------- /_testdata/05893125684f2d3943cd84a7ab2b75e53668fba1.siva: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/src-d/gitbase/c9a16f3b6d8171b595570787fbc2170325c08b48/_testdata/05893125684f2d3943cd84a7ab2b75e53668fba1.siva -------------------------------------------------------------------------------- /_testdata/fff7062de8474d10a67d417ccea87ba6f58ca81d.siva: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/src-d/gitbase/c9a16f3b6d8171b595570787fbc2170325c08b48/_testdata/fff7062de8474d10a67d417ccea87ba6f58ca81d.siva -------------------------------------------------------------------------------- /_testdata/fff840f8784ef162dc83a1465fc5763d890b68ba.siva: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/src-d/gitbase/c9a16f3b6d8171b595570787fbc2170325c08b48/_testdata/fff840f8784ef162dc83a1465fc5763d890b68ba.siva -------------------------------------------------------------------------------- /_testdata/library.yaml: -------------------------------------------------------------------------------- 1 | id: d4213320-c4f0-11e9-b018-9cb6d0e2bf3b 2 | version: -1 3 | -------------------------------------------------------------------------------- /_testdata/not-permission/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/src-d/gitbase/c9a16f3b6d8171b595570787fbc2170325c08b48/_testdata/not-permission/.gitkeep -------------------------------------------------------------------------------- /_testdata/not-siva.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/src-d/gitbase/c9a16f3b6d8171b595570787fbc2170325c08b48/_testdata/not-siva.txt -------------------------------------------------------------------------------- /_testdata/regression.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - 3 | ID: 'query00' 4 | Name: 'All commits' 5 | Statements: 6 | - SELECT * FROM commits 7 | - 8 | ID: 'query01' 9 | Name: 'Last commit messages in HEAD for every repository' 10 | Statements: 11 | - SELECT c.commit_message FROM refs r JOIN commits c ON r.commit_hash = c.commit_hash WHERE r.ref_name = 'HEAD' 12 | - 13 | ID: 'query02' 14 | Name: 'All commit messages in HEAD history for every repository' 15 | Statements: 16 | - SELECT c.commit_message FROM commits c NATURAL JOIN ref_commits r WHERE r.ref_name = 'HEAD' 17 | - 18 | ID: 'query03' 19 | Name: 'Top 10 repositories by commit count in HEAD' 20 | Statements: 21 | - SELECT repository_id,commit_count FROM (SELECT r.repository_id,count(*) AS commit_count FROM ref_commits r WHERE r.ref_name = 'HEAD' GROUP BY r.repository_id) AS q ORDER BY commit_count DESC LIMIT 10 22 | - 23 | ID: 'query04' 24 | Name: 'Top 10 repositories by contributor count (all branches)' 25 | Statements: 26 | - > 27 | SELECT 28 | repository_id, 29 | COUNT(commit_author_email) as contributor_count 30 | FROM ( 31 | SELECT DISTINCT 32 | repository_id, 33 | commit_author_email 34 | FROM commits 35 | ) as q 36 | GROUP BY repository_id 37 | ORDER BY contributor_count DESC 38 | LIMIT 10 39 | - 40 | ID: 'query05' 41 | Name: 'Query all files from HEAD' 42 | Statements: 43 | - SELECT cf.file_path, f.blob_content FROM ref_commits r NATURAL JOIN commit_files cf NATURAL JOIN files f WHERE r.ref_name = 'HEAD' AND r.history_index = 0 44 | - 45 | ID: 'query06' 46 | Name: '10 top repos by file count in HEAD' 47 | Statements: 48 | - SELECT repository_id, num_files FROM (SELECT COUNT(f.*) num_files, f.repository_id FROM ref_commits r INNER JOIN commit_files cf ON r.commit_hash = cf.commit_hash AND r.repository_id = cf.repository_id INNER JOIN files f ON cf.repository_id = f.repository_id AND cf.blob_hash = f.blob_hash AND cf.tree_hash = f.tree_hash AND cf.file_path = f.file_path WHERE r.ref_name = 'HEAD' GROUP BY f.repository_id) t ORDER BY num_files DESC LIMIT 10 49 | - 50 | ID: 'query07' 51 | Name: 'Top committers per repository' 52 | Statements: 53 | - SELECT * FROM (SELECT commit_author_email as author, repository_id as id, count(*) as num_commits FROM commits GROUP BY commit_author_email, repository_id) AS t ORDER BY num_commits DESC 54 | - 55 | ID: 'query08' 56 | Name: 'Top committers in all repositories' 57 | Statements: 58 | - SELECT * FROM (SELECT commit_author_email as author,count(*) as num_commits FROM commits GROUP BY commit_author_email) t ORDER BY num_commits DESC 59 | - 60 | ID: 'query09' 61 | Name: 'Count all commits with NOT operation' 62 | Statements: 63 | - SELECT COUNT(*) FROM commits WHERE NOT(commit_author_email = 'non existing email address'); 64 | - 65 | ID: 'query10' 66 | Name: 'Count all commits with NOT operation and pilosa index' 67 | Statements: 68 | - CREATE INDEX email_idx ON commits USING pilosa (commit_author_email) WITH (async = false) 69 | - SELECT COUNT(*) FROM commits WHERE NOT(commit_author_email = 'non existing email address') 70 | - DROP INDEX email_idx ON commits 71 | - 72 | ID: 'query11' 73 | Name: 'Leak detection on all refs' 74 | Statements: 75 | - > 76 | SELECT f.repository_id, f.blob_hash, f.commit_hash, f.file_path 77 | FROM ( 78 | SELECT blob_hash, repository_id 79 | FROM blobs 80 | WHERE NOT IS_BINARY(blob_content) AND ( 81 | blob_content REGEXP '(?i)facebook.*[\'\\"][0-9a-f]{32}[\'\\"]' 82 | OR blob_content REGEXP '(?i)twitter.*[\'\\"][0-9a-zA-Z]{35,44}[\'\\"]' 83 | OR blob_content REGEXP '(?i)github.*[\'\\"][0-9a-zA-Z]{35,40}[\'\\"]' 84 | OR blob_content REGEXP 'AKIA[0-9A-Z]{16}' 85 | OR blob_content REGEXP '(?i)heroku.*[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}' 86 | OR blob_content REGEXP '.*-----BEGIN ((RSA|DSA|OPENSSH|SSH2|EC) )?PRIVATE KEY-----.*' 87 | ) 88 | ) h 89 | INNER JOIN commit_files f 90 | ON h.blob_hash = f.blob_hash 91 | AND h.repository_id = f.repository_id 92 | AND f.file_path NOT REGEXP '^vendor.*' 93 | - 94 | ID: 'query12' 95 | Name: 'Leaks detection on HEAD' 96 | Statements: 97 | - > 98 | SELECT repository_id, blob_hash, commit_hash, file_path 99 | FROM refs r 100 | NATURAL JOIN commit_files cf 101 | NATURAL JOIN files f 102 | WHERE r.ref_name = 'HEAD' AND NOT IS_BINARY(blob_content) AND ( 103 | blob_content REGEXP '(?i)facebook.*[\'\\"][0-9a-f]{32}[\'\\"]' 104 | OR blob_content REGEXP '(?i)twitter.*[\'\\"][0-9a-zA-Z]{35,44}[\'\\"]' 105 | OR blob_content REGEXP '(?i)github.*[\'\\"][0-9a-zA-Z]{35,40}[\'\\"]' 106 | OR blob_content REGEXP 'AKIA[0-9A-Z]{16}' 107 | OR blob_content REGEXP '(?i)heroku.*[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}' 108 | OR blob_content REGEXP '.*-----BEGIN ((RSA|DSA|OPENSSH|SSH2|EC) )?PRIVATE KEY-----.*' 109 | ) AND cf.file_path NOT REGEXP '^vendor.*' 110 | - 111 | ID: 'query13' 112 | Name: 'Has README' 113 | Statements: 114 | - > 115 | SELECT DISTINCT repository_id 116 | FROM refs r 117 | NATURAL JOIN commits 118 | NATURAL JOIN tree_entries te 119 | WHERE r.ref_name = 'HEAD' 120 | AND te.tree_entry_name REGEXP '^(?i)readme(\.[a-z]+)?$' 121 | -------------------------------------------------------------------------------- /_testdata/repositories-link: -------------------------------------------------------------------------------- 1 | ./repositories -------------------------------------------------------------------------------- /_testdata/repositories/repositories-link: -------------------------------------------------------------------------------- 1 | repositories-link -------------------------------------------------------------------------------- /blobs_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/src-d/go-mysql-server/sql" 7 | "github.com/src-d/go-mysql-server/sql/expression" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestBlobsTable(t *testing.T) { 12 | require := require.New(t) 13 | ctx, _, cleanup := setup(t) 14 | defer cleanup() 15 | 16 | table := getTable(t, BlobsTableName, ctx) 17 | 18 | rows, err := tableToRows(ctx, table) 19 | require.NoError(err) 20 | require.Len(rows, 10) 21 | 22 | schema := table.Schema() 23 | for idx, row := range rows { 24 | err := schema.CheckRow(row) 25 | require.NoError(err, "row %d doesn't conform to schema", idx) 26 | } 27 | } 28 | 29 | func TestBlobsLimit(t *testing.T) { 30 | require := require.New(t) 31 | ctx, _, cleanup := setup(t) 32 | defer cleanup() 33 | 34 | prev := blobsMaxSize 35 | blobsMaxSize = 200000 36 | defer func() { 37 | blobsMaxSize = prev 38 | }() 39 | 40 | table := newBlobsTable(poolFromCtx(t, ctx)). 41 | WithProjection([]string{"blob_content"}) 42 | rows, err := tableToRows(ctx, table) 43 | require.NoError(err) 44 | 45 | expected := []struct { 46 | hash string 47 | bytes int64 48 | empty bool 49 | }{ 50 | {"32858aad3c383ed1ff0a0f9bdf231d54a00c9e88", 189, false}, 51 | {"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", 18, false}, 52 | {"c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", 1072, false}, 53 | {"7e59600739c96546163833214c36459e324bad0a", 9, false}, 54 | {"d5c0f4ab811897cadf03aec358ae60d21f91c50d", 76110, true}, // is binary 55 | {"880cd14280f4b9b6ed3986d6671f907d7cc2a198", 2780, false}, 56 | {"49c6bb89b17060d7b4deacb7b338fcc6ea2352a9", 217848, true}, // exceeds threshold 57 | {"c8f1d8c61f9da76f4cb49fd86322b6e685dba956", 706, false}, 58 | {"9a48f23120e880dfbe41f7c9b7b708e9ee62a492", 11488, false}, 59 | {"9dea2395f5403188298c1dabe8bdafe562c491e3", 78, false}, 60 | } 61 | 62 | require.Len(rows, len(expected)) 63 | for i, row := range rows { 64 | e := expected[i] 65 | require.Equal(e.hash, row[1].(string)) 66 | require.Equal(e.bytes, row[2].(int64)) 67 | require.Equal(e.empty, len(row[3].([]byte)) == 0) 68 | } 69 | } 70 | 71 | func TestBlobsPushdown(t *testing.T) { 72 | require := require.New(t) 73 | ctx, _, cleanup := setup(t) 74 | defer cleanup() 75 | 76 | table := newBlobsTable(poolFromCtx(t, ctx)) 77 | 78 | rows, err := tableToRows(ctx, table) 79 | require.NoError(err) 80 | require.Len(rows, 10) 81 | 82 | t2 := table.WithFilters([]sql.Expression{ 83 | expression.NewEquals( 84 | expression.NewGetFieldWithTable(1, sql.Text, BlobsTableName, "blob_hash", false), 85 | expression.NewLiteral("32858aad3c383ed1ff0a0f9bdf231d54a00c9e88", sql.Text), 86 | ), 87 | }) 88 | 89 | rows, err = tableToRows(ctx, t2) 90 | require.NoError(err) 91 | require.Len(rows, 1) 92 | 93 | t3 := table.WithFilters([]sql.Expression{ 94 | expression.NewEquals( 95 | expression.NewGetFieldWithTable(1, sql.Text, BlobsTableName, "blob_hash", false), 96 | expression.NewLiteral("not exists", sql.Text), 97 | ), 98 | }) 99 | 100 | rows, err = tableToRows(ctx, t3) 101 | require.NoError(err) 102 | require.Len(rows, 0) 103 | } 104 | 105 | func TestBlobsIndexKeyValueIter(t *testing.T) { 106 | require := require.New(t) 107 | ctx, path, cleanup := setup(t) 108 | defer cleanup() 109 | 110 | table := new(blobsTable) 111 | iter, err := table.IndexKeyValues(ctx, []string{"blob_hash", "blob_size"}) 112 | require.NoError(err) 113 | 114 | var expected = []keyValue{ 115 | { 116 | assertEncodeKey(t, &packOffsetIndexKey{ 117 | Repository: path, 118 | Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa", 119 | Offset: 1591, 120 | }), 121 | []interface{}{ 122 | "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88", 123 | int64(189), 124 | }, 125 | }, 126 | { 127 | assertEncodeKey(t, &packOffsetIndexKey{ 128 | Repository: path, 129 | Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa", 130 | Offset: 79864, 131 | }), 132 | []interface{}{ 133 | "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9", 134 | int64(217848), 135 | }, 136 | }, 137 | { 138 | assertEncodeKey(t, &packOffsetIndexKey{ 139 | Repository: path, 140 | Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa", 141 | Offset: 2418, 142 | }), 143 | []interface{}{ 144 | "7e59600739c96546163833214c36459e324bad0a", 145 | int64(9), 146 | }, 147 | }, 148 | { 149 | assertEncodeKey(t, &packOffsetIndexKey{ 150 | Repository: path, 151 | Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa", 152 | Offset: 78932, 153 | }), 154 | []interface{}{ 155 | "880cd14280f4b9b6ed3986d6671f907d7cc2a198", 156 | int64(2780), 157 | }, 158 | }, 159 | { 160 | assertEncodeKey(t, &packOffsetIndexKey{ 161 | Repository: path, 162 | Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa", 163 | Offset: 82000, 164 | }), 165 | []interface{}{ 166 | "9a48f23120e880dfbe41f7c9b7b708e9ee62a492", 167 | int64(11488), 168 | }, 169 | }, 170 | { 171 | assertEncodeKey(t, &packOffsetIndexKey{ 172 | Repository: path, 173 | Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa", 174 | Offset: 85438, 175 | }), 176 | []interface{}{ 177 | "9dea2395f5403188298c1dabe8bdafe562c491e3", 178 | int64(78), 179 | }, 180 | }, 181 | { 182 | assertEncodeKey(t, &packOffsetIndexKey{ 183 | Repository: path, 184 | Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa", 185 | Offset: 1780, 186 | }), 187 | []interface{}{ 188 | "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", 189 | int64(1072), 190 | }, 191 | }, 192 | { 193 | assertEncodeKey(t, &packOffsetIndexKey{ 194 | Repository: path, 195 | Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa", 196 | Offset: 81707, 197 | }), 198 | []interface{}{ 199 | "c8f1d8c61f9da76f4cb49fd86322b6e685dba956", 200 | int64(706), 201 | }, 202 | }, 203 | { 204 | assertEncodeKey(t, &packOffsetIndexKey{ 205 | Repository: path, 206 | Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa", 207 | Offset: 1752, 208 | }), 209 | []interface{}{ 210 | "d3ff53e0564a9f87d8e84b6e28e5060e517008aa", 211 | int64(18), 212 | }, 213 | }, 214 | { 215 | assertEncodeKey(t, &packOffsetIndexKey{ 216 | Repository: path, 217 | Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa", 218 | Offset: 2436, 219 | }), 220 | []interface{}{ 221 | "d5c0f4ab811897cadf03aec358ae60d21f91c50d", 222 | int64(76110), 223 | }, 224 | }, 225 | } 226 | 227 | assertIndexKeyValueIter(t, iter, expected) 228 | } 229 | 230 | func TestBlobsIndex(t *testing.T) { 231 | testTableIndex( 232 | t, 233 | new(blobsTable), 234 | []sql.Expression{expression.NewEquals( 235 | expression.NewGetField(1, sql.Text, "commit_hash", false), 236 | expression.NewLiteral("af2d6a6954d532f8ffb47615169c8fdf9d383a1a", sql.Text), 237 | )}, 238 | ) 239 | } 240 | 241 | func TestBlobsIndexIterClosed(t *testing.T) { 242 | testTableIndexIterClosed(t, new(blobsTable)) 243 | } 244 | 245 | func TestBlobsIterClosed(t *testing.T) { 246 | testTableIterClosed(t, new(blobsTable)) 247 | } 248 | -------------------------------------------------------------------------------- /checksum.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha1" 6 | "encoding/base64" 7 | "io" 8 | "sort" 9 | "strings" 10 | 11 | "gopkg.in/src-d/go-git.v4/plumbing" 12 | ) 13 | 14 | type checksumable struct { 15 | pool *RepositoryPool 16 | } 17 | 18 | func (c *checksumable) Checksum() (string, error) { 19 | hash := sha1.New() 20 | iter, err := c.pool.RepoIter() 21 | if err != nil { 22 | return "", err 23 | } 24 | defer iter.Close() 25 | 26 | var checksums checksums 27 | for { 28 | hash.Reset() 29 | 30 | repo, err := iter.Next() 31 | if err == io.EOF { 32 | break 33 | } 34 | if err != nil { 35 | return "", err 36 | } 37 | 38 | bytes, err := readChecksum(repo) 39 | if err != nil { 40 | return "", err 41 | } 42 | 43 | if _, err = hash.Write(bytes); err != nil { 44 | return "", err 45 | } 46 | 47 | bytes, err = readRefs(repo) 48 | if err != nil { 49 | return "", err 50 | } 51 | 52 | if _, err = hash.Write(bytes); err != nil { 53 | return "", err 54 | } 55 | 56 | c := checksum{ 57 | name: repo.ID(), 58 | hash: hash.Sum(nil), 59 | } 60 | 61 | checksums = append(checksums, c) 62 | } 63 | 64 | sort.Stable(checksums) 65 | hash.Reset() 66 | 67 | for _, c := range checksums { 68 | if _, err = hash.Write(c.hash); err != nil { 69 | return "", err 70 | } 71 | } 72 | 73 | return base64.StdEncoding.EncodeToString(hash.Sum(nil)), nil 74 | } 75 | 76 | func readChecksum(r *Repository) ([]byte, error) { 77 | fs, err := r.FS() 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | dot, packfiles, err := repositoryPackfiles(fs) 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | var result []byte 88 | for _, p := range packfiles { 89 | f, err := dot.ObjectPack(p) 90 | if err != nil { 91 | return nil, err 92 | } 93 | 94 | if _, err = f.Seek(-20, io.SeekEnd); err != nil { 95 | return nil, err 96 | } 97 | 98 | var checksum = make([]byte, 20) 99 | if _, err = io.ReadFull(f, checksum); err != nil { 100 | return nil, err 101 | } 102 | 103 | if err = f.Close(); err != nil { 104 | return nil, err 105 | } 106 | 107 | result = append(result, checksum...) 108 | } 109 | 110 | return result, nil 111 | } 112 | 113 | type reference struct { 114 | name string 115 | hash string 116 | } 117 | 118 | type references []reference 119 | 120 | type byHashAndName []reference 121 | 122 | func (b byHashAndName) Len() int { return len(b) } 123 | func (b byHashAndName) Swap(i, j int) { b[i], b[j] = b[j], b[i] } 124 | func (b byHashAndName) Less(i, j int) bool { 125 | if cmp := strings.Compare(b[i].hash, b[j].hash); cmp != 0 { 126 | return cmp < 0 127 | } 128 | return strings.Compare(b[i].name, b[j].name) < 0 129 | } 130 | 131 | type checksum struct { 132 | name string 133 | hash []byte 134 | } 135 | 136 | type checksums []checksum 137 | 138 | func (b checksums) Len() int { return len(b) } 139 | func (b checksums) Swap(i, j int) { b[i], b[j] = b[j], b[i] } 140 | func (b checksums) Less(i, j int) bool { 141 | if cmp := bytes.Compare(b[i].hash, b[j].hash); cmp != 0 { 142 | return cmp < 0 143 | } 144 | return strings.Compare(b[i].name, b[j].name) < 0 145 | } 146 | 147 | func readRefs(repo *Repository) ([]byte, error) { 148 | buf := bytes.NewBuffer(nil) 149 | 150 | refs, err := repo.References() 151 | if err != nil { 152 | return nil, err 153 | } 154 | 155 | var references []reference 156 | err = refs.ForEach(func(r *plumbing.Reference) error { 157 | references = append(references, reference{ 158 | name: string(r.Name()), 159 | hash: r.Hash().String(), 160 | }) 161 | return nil 162 | }) 163 | if err != nil { 164 | return nil, err 165 | } 166 | 167 | sort.Stable(byHashAndName(references)) 168 | 169 | for _, r := range references { 170 | buf.WriteString(r.name) 171 | buf.WriteString(r.hash) 172 | } 173 | 174 | return buf.Bytes(), nil 175 | } 176 | -------------------------------------------------------------------------------- /checksum_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | 9 | fixtures "github.com/src-d/go-git-fixtures" 10 | "github.com/stretchr/testify/require" 11 | "gopkg.in/src-d/go-billy.v4/osfs" 12 | ) 13 | 14 | const ( 15 | checksumMulti = "W/lxpR0jZ6O6BqVANTYTDMlAS/4=" 16 | checksumSingle = "zqLF31JlrtJ57XNC+cQ+2hSkBkw=" 17 | checksumSiva = "X27U+Lww5UOk1+/21bVFgI4uJyM=" 18 | ) 19 | 20 | func TestChecksum(t *testing.T) { 21 | require := require.New(t) 22 | 23 | defer func() { 24 | require.NoError(fixtures.Clean()) 25 | }() 26 | 27 | lib, pool, err := newMultiPool() 28 | require.NoError(err) 29 | 30 | for i, f := range fixtures.ByTag("worktree") { 31 | path := f.Worktree().Root() 32 | require.NoError(lib.AddPlain(fmt.Sprintf("repo_%d", i), path, nil)) 33 | } 34 | 35 | c := &checksumable{pool} 36 | checksum, err := c.Checksum() 37 | require.NoError(err) 38 | require.Equal(checksumMulti, checksum) 39 | 40 | lib, pool, err = newMultiPool() 41 | require.NoError(err) 42 | path := fixtures.ByTag("worktree").One().Worktree().Root() 43 | require.NoError(lib.AddPlain("worktree", path, nil)) 44 | 45 | c = &checksumable{pool} 46 | checksum, err = c.Checksum() 47 | require.NoError(err) 48 | require.Equal(checksumSingle, checksum) 49 | } 50 | 51 | func TestChecksumSiva(t *testing.T) { 52 | require := require.New(t) 53 | 54 | lib, pool, err := newMultiPool() 55 | require.NoError(err) 56 | 57 | cwd, err := os.Getwd() 58 | require.NoError(err) 59 | cwdFS := osfs.New(cwd) 60 | 61 | require.NoError( 62 | filepath.Walk("_testdata", func(path string, info os.FileInfo, err error) error { 63 | if err != nil { 64 | return err 65 | } 66 | 67 | if IsSivaFile(path) { 68 | require.NoError(lib.AddSiva(path, cwdFS)) 69 | } 70 | 71 | return nil 72 | }), 73 | ) 74 | 75 | c := &checksumable{pool} 76 | checksum, err := c.Checksum() 77 | require.NoError(err) 78 | require.Equal(checksumSiva, checksum) 79 | } 80 | 81 | func TestChecksumStable(t *testing.T) { 82 | require := require.New(t) 83 | 84 | defer func() { 85 | require.NoError(fixtures.Clean()) 86 | }() 87 | 88 | lib, pool, err := newMultiPool() 89 | require.NoError(err) 90 | 91 | for i, f := range fixtures.ByTag("worktree") { 92 | path := f.Worktree().Root() 93 | require.NoError(lib.AddPlain(fmt.Sprintf("repo_%d", i), path, nil)) 94 | } 95 | 96 | c := &checksumable{pool} 97 | 98 | for i := 0; i < 100; i++ { 99 | checksum, err := c.Checksum() 100 | require.NoError(err) 101 | require.Equal(checksumMulti, checksum) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /cmd/gitbase/command/version.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import "fmt" 4 | 5 | const ( 6 | VersionDescription = "Show the version information" 7 | VersionHelp = VersionDescription 8 | ) 9 | 10 | // Version represents the `version` command of gitbase cli tool. 11 | type Version struct { 12 | // Name of the cli binary 13 | Name string 14 | // Version of the cli binary 15 | Version string 16 | // Build of the cli binary 17 | Build string 18 | } 19 | 20 | // Execute prints the build information provided by the compilation tools, it 21 | // honors the go-flags.Commander interface. 22 | func (c *Version) Execute(args []string) error { 23 | fmt.Printf("%s (%s) - build %s\n", c.Name, c.Version, c.Build) 24 | return nil 25 | } 26 | -------------------------------------------------------------------------------- /cmd/gitbase/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/src-d/gitbase/cmd/gitbase/command" 8 | 9 | "github.com/jessevdk/go-flags" 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | var ( 14 | name = "gitbase" 15 | version = "undefined" 16 | build = "undefined" 17 | ) 18 | 19 | func main() { 20 | parser := flags.NewNamedParser(name, flags.Default) 21 | parser.UnknownOptionHandler = func(option string, arg flags.SplitArgument, args []string) ([]string, error) { 22 | if option != "g" { 23 | return nil, fmt.Errorf("unknown flag `%s'", option) 24 | } 25 | 26 | if len(args) == 0 { 27 | return nil, fmt.Errorf("unknown flag `%s'", option) 28 | } 29 | 30 | return append(append(args, "-d"), args[0]), nil 31 | } 32 | 33 | _, err := parser.AddCommand("server", command.ServerDescription, command.ServerHelp, 34 | &command.Server{ 35 | SkipGitErrors: os.Getenv("GITBASE_SKIP_GIT_ERRORS") != "", 36 | Version: version, 37 | }) 38 | if err != nil { 39 | logrus.Fatal(err) 40 | } 41 | 42 | _, err = parser.AddCommand("version", command.VersionDescription, command.VersionHelp, 43 | &command.Version{ 44 | Name: name, 45 | Version: version, 46 | Build: build, 47 | }) 48 | if err != nil { 49 | logrus.Fatal(err) 50 | } 51 | 52 | _, err = parser.Parse() 53 | if err != nil { 54 | if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrCommandRequired { 55 | parser.WriteHelp(os.Stdout) 56 | } 57 | 58 | os.Exit(1) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /commit_files_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "io" 5 | "strconv" 6 | "testing" 7 | 8 | "github.com/src-d/go-mysql-server/sql" 9 | "github.com/src-d/go-mysql-server/sql/expression" 10 | "github.com/stretchr/testify/require" 11 | "gopkg.in/src-d/go-git.v4/plumbing" 12 | ) 13 | 14 | func TestCommitFilesTableRowIter(t *testing.T) { 15 | require := require.New(t) 16 | 17 | ctx, _, cleanup := setupRepos(t) 18 | defer cleanup() 19 | 20 | table := newCommitFilesTable(poolFromCtx(t, ctx)) 21 | require.NotNil(table) 22 | 23 | rows, err := tableToRows(ctx, table) 24 | require.NoError(err) 25 | 26 | var expected []sql.Row 27 | s, err := getSession(ctx) 28 | require.NoError(err) 29 | repos, err := s.Pool.RepoIter() 30 | require.NoError(err) 31 | for { 32 | repo, err := repos.Next() 33 | if err == io.EOF { 34 | break 35 | } 36 | 37 | require.NoError(err) 38 | 39 | commits, err := newCommitIter(repo, false) 40 | require.NoError(err) 41 | 42 | for { 43 | commit, err := commits.Next() 44 | if err == io.EOF { 45 | break 46 | } 47 | 48 | require.NoError(err) 49 | 50 | fi, err := commit.Files() 51 | require.NoError(err) 52 | 53 | for { 54 | f, err := fi.Next() 55 | if err == io.EOF { 56 | break 57 | } 58 | 59 | require.NoError(err) 60 | 61 | expected = append(expected, newCommitFilesRow(repo, commit, f)) 62 | } 63 | } 64 | } 65 | 66 | require.ElementsMatch(expected, rows) 67 | } 68 | 69 | func TestCommitFilesIndex(t *testing.T) { 70 | testTableIndex( 71 | t, 72 | new(commitFilesTable), 73 | []sql.Expression{expression.NewEquals( 74 | expression.NewGetField(1, sql.Text, "commit_hash", false), 75 | expression.NewLiteral("af2d6a6954d532f8ffb47615169c8fdf9d383a1a", sql.Text), 76 | )}, 77 | ) 78 | } 79 | 80 | func TestCommitFilesOr(t *testing.T) { 81 | testTableIndex( 82 | t, 83 | new(commitFilesTable), 84 | []sql.Expression{ 85 | expression.NewOr( 86 | expression.NewEquals( 87 | expression.NewGetField(1, sql.Text, "commit_hash", false), 88 | expression.NewLiteral("1669dce138d9b841a518c64b10914d88f5e488ea", sql.Text), 89 | ), 90 | expression.NewEquals( 91 | expression.NewGetField(2, sql.Text, "file_path", false), 92 | expression.NewLiteral("go/example.go", sql.Text), 93 | ), 94 | ), 95 | }, 96 | ) 97 | } 98 | 99 | func TestEncodeCommitFileIndexKey(t *testing.T) { 100 | require := require.New(t) 101 | 102 | k := commitFileIndexKey{ 103 | Repository: "repo1", 104 | Packfile: plumbing.ZeroHash.String(), 105 | Offset: 1234, 106 | Hash: plumbing.ZeroHash.String(), 107 | Name: "foo/bar.md", 108 | Mode: 5, 109 | Tree: plumbing.ZeroHash.String(), 110 | Commit: plumbing.ZeroHash.String(), 111 | } 112 | 113 | data, err := k.encode() 114 | require.NoError(err) 115 | 116 | var k2 commitFileIndexKey 117 | require.NoError(k2.decode(data)) 118 | 119 | require.Equal(k, k2) 120 | } 121 | 122 | func TestCommitFilesIndexIterClosed(t *testing.T) { 123 | testTableIndexIterClosed(t, new(commitFilesTable)) 124 | } 125 | 126 | func TestCommitFilesIterClosed(t *testing.T) { 127 | testTableIterClosed(t, new(commitFilesTable)) 128 | } 129 | 130 | func TestPartitionRowsWithIndex(t *testing.T) { 131 | require := require.New(t) 132 | ctx, _, cleanup := setup(t) 133 | defer cleanup() 134 | 135 | table := new(commitFilesTable) 136 | expected, err := tableToRows(ctx, table) 137 | require.NoError(err) 138 | 139 | lookup := tableIndexLookup(t, table, ctx) 140 | tbl := table.WithIndexLookup(lookup) 141 | 142 | result, err := tableToRows(ctx, tbl) 143 | require.NoError(err) 144 | 145 | require.ElementsMatch(expected, result) 146 | } 147 | 148 | func TestCommitFilesIndexIter(t *testing.T) { 149 | require := require.New(t) 150 | 151 | ctx, _, cleanup := setupRepos(t) 152 | defer cleanup() 153 | 154 | key := &commitFileIndexKey{ 155 | Repository: "zero", 156 | Packfile: plumbing.ZeroHash.String(), 157 | Hash: plumbing.ZeroHash.String(), 158 | Offset: 0, 159 | Name: "two", 160 | Mode: 5, 161 | Tree: plumbing.ZeroHash.String(), 162 | Commit: plumbing.ZeroHash.String(), 163 | } 164 | limit := 10 165 | it := newCommitFilesIndexIter(testIndexValueIter{key, int64(limit)}, poolFromCtx(t, ctx)) 166 | for off := 0; off < limit; off++ { 167 | row, err := it.Next() 168 | require.NoError(err) 169 | 170 | require.Equal(key.Repository, row[0]) 171 | require.Equal(strconv.Itoa(off), row[2]) 172 | } 173 | _, err := it.Next() 174 | require.EqualError(err, io.EOF.Error()) 175 | } 176 | 177 | type testIndexValueIter struct { 178 | key *commitFileIndexKey 179 | limit int64 180 | } 181 | 182 | func (it testIndexValueIter) Next() ([]byte, error) { 183 | if it.key.Offset >= it.limit { 184 | return nil, io.EOF 185 | } 186 | 187 | it.key.Name = strconv.Itoa(int(it.key.Offset)) 188 | val, err := it.key.encode() 189 | if err != nil { 190 | return nil, err 191 | } 192 | val, err = encoder.encode(val) 193 | if err != nil { 194 | return nil, err 195 | } 196 | 197 | it.key.Offset++ 198 | return val, nil 199 | } 200 | 201 | func (it testIndexValueIter) Close() error { 202 | return nil 203 | } 204 | -------------------------------------------------------------------------------- /commit_trees_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/src-d/go-mysql-server/sql" 7 | "github.com/src-d/go-mysql-server/sql/expression" 8 | "github.com/stretchr/testify/require" 9 | "gopkg.in/src-d/go-git.v4/plumbing" 10 | ) 11 | 12 | func TestCommitTreesRowIter(t *testing.T) { 13 | require := require.New(t) 14 | ctx, _, cleanup := setup(t) 15 | defer cleanup() 16 | 17 | rows, err := tableToRows(ctx, new(commitTreesTable)) 18 | require.NoError(err) 19 | 20 | for i, row := range rows { 21 | // remove repository ids 22 | rows[i] = row[1:] 23 | } 24 | 25 | expected := []sql.Row{ 26 | {"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "a8d315b2b1c615d43042c3a62402b8a54288cf5c"}, 27 | {"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "a39771a7651f97faf5c72e08224d857fc35133db"}, 28 | {"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"}, 29 | {"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"}, 30 | {"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "cf4aa3b38974fb7d81f367c0830f7d78d65ab86b"}, 31 | 32 | {"e8d3ffab552895c19b9fcf7aa264d277cde33881", "dbd3641b371024f44d0e469a9c8f5457b0660de1"}, 33 | {"e8d3ffab552895c19b9fcf7aa264d277cde33881", "a39771a7651f97faf5c72e08224d857fc35133db"}, 34 | {"e8d3ffab552895c19b9fcf7aa264d277cde33881", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"}, 35 | {"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"}, 36 | 37 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "fb72698cab7617ac416264415f13224dfd7a165e"}, 38 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "a39771a7651f97faf5c72e08224d857fc35133db"}, 39 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"}, 40 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"}, 41 | 42 | {"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd"}, 43 | {"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"}, 44 | 45 | {"1669dce138d9b841a518c64b10914d88f5e488ea", "eba74343e2f15d62adedfd8c883ee0262b5c8021"}, 46 | 47 | {"35e85108805c84807bc66a02d91535e1e24b38b9", "8dcef98b1d52143e1e2dbc458ffe38f925786bf2"}, 48 | 49 | {"b029517f6300c2da0f4b651b8642506cd6aaf45d", "aa9b383c260e1d05fbbf6b30a02914555e20c725"}, 50 | 51 | {"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"}, 52 | 53 | {"b8e471f58bcbca63b07bda20e428190409c2db47", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"}, 54 | } 55 | 56 | require.ElementsMatch(expected, rows) 57 | } 58 | 59 | func TestCommitTreesPushdown(t *testing.T) { 60 | ctx, _, cleanup := setup(t) 61 | defer cleanup() 62 | 63 | table := new(commitTreesTable) 64 | testCases := []struct { 65 | name string 66 | filters []sql.Expression 67 | expected []sql.Row 68 | }{ 69 | { 70 | "commit filter", 71 | []sql.Expression{ 72 | expression.NewEquals( 73 | expression.NewGetFieldWithTable(1, sql.Text, CommitTreesTableName, "commit_hash", false), 74 | expression.NewLiteral("918c48b83bd081e863dbe1b80f8998f058cd8294", sql.Text), 75 | ), 76 | }, 77 | []sql.Row{ 78 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "fb72698cab7617ac416264415f13224dfd7a165e"}, 79 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "a39771a7651f97faf5c72e08224d857fc35133db"}, 80 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"}, 81 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"}, 82 | }, 83 | }, 84 | { 85 | "tree filter", 86 | []sql.Expression{ 87 | expression.NewEquals( 88 | expression.NewGetFieldWithTable(2, sql.Text, CommitTreesTableName, "tree_hash", false), 89 | expression.NewLiteral("586af567d0bb5e771e49bdd9434f5e0fb76d25fa", sql.Text), 90 | ), 91 | }, 92 | []sql.Row{ 93 | {"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"}, 94 | {"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"}, 95 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"}, 96 | }, 97 | }, 98 | } 99 | 100 | for _, tt := range testCases { 101 | t.Run(tt.name, func(t *testing.T) { 102 | require := require.New(t) 103 | tbl := table.WithFilters(tt.filters) 104 | rows, err := tableToRows(ctx, tbl) 105 | require.NoError(err) 106 | 107 | for i, row := range rows { 108 | // remove repository_ids 109 | rows[i] = row[1:] 110 | } 111 | 112 | require.ElementsMatch(tt.expected, rows) 113 | }) 114 | } 115 | } 116 | 117 | func TestCommitTreesIndexKeyValueIter(t *testing.T) { 118 | require := require.New(t) 119 | ctx, _, cleanup := setup(t) 120 | defer cleanup() 121 | 122 | table := new(commitTreesTable) 123 | iter, err := table.IndexKeyValues(ctx, []string{"tree_hash", "commit_hash"}) 124 | require.NoError(err) 125 | 126 | rows, err := tableToRows(ctx, table) 127 | require.NoError(err) 128 | 129 | var expected []keyValue 130 | for _, row := range rows { 131 | var kv keyValue 132 | kv.key = assertEncodeCommitTreesRow(t, row) 133 | kv.values = append(kv.values, row[2], row[1]) 134 | expected = append(expected, kv) 135 | } 136 | 137 | assertIndexKeyValueIter(t, iter, expected) 138 | } 139 | 140 | func assertEncodeCommitTreesRow(t *testing.T, row sql.Row) []byte { 141 | t.Helper() 142 | k, err := new(commitTreesRowKeyMapper).fromRow(row) 143 | require.NoError(t, err) 144 | return k 145 | } 146 | 147 | func TestCommitTreesIndex(t *testing.T) { 148 | testTableIndex( 149 | t, 150 | new(commitTreesTable), 151 | []sql.Expression{expression.NewEquals( 152 | expression.NewGetField(1, sql.Text, "commit_hash", false), 153 | expression.NewLiteral("af2d6a6954d532f8ffb47615169c8fdf9d383a1a", sql.Text), 154 | )}, 155 | ) 156 | } 157 | 158 | func TestCommitTreesRowKeyMapper(t *testing.T) { 159 | require := require.New(t) 160 | row := sql.Row{"repo1", plumbing.ZeroHash.String(), plumbing.ZeroHash.String()} 161 | mapper := new(commitTreesRowKeyMapper) 162 | 163 | k, err := mapper.fromRow(row) 164 | require.NoError(err) 165 | 166 | row2, err := mapper.toRow(k) 167 | require.NoError(err) 168 | 169 | require.Equal(row, row2) 170 | } 171 | 172 | func TestCommitTreesIndexIterClosed(t *testing.T) { 173 | testTableIndexIterClosed(t, new(commitTreesTable)) 174 | } 175 | 176 | func TestCommitTreesIterClosed(t *testing.T) { 177 | testTableIterClosed(t, new(commitTreesTable)) 178 | } 179 | -------------------------------------------------------------------------------- /database.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "github.com/src-d/go-mysql-server/sql" 5 | ) 6 | 7 | const ( 8 | // ReferencesTableName is the name of the refs table. 9 | ReferencesTableName = "refs" 10 | // CommitsTableName is the name of the commits table. 11 | CommitsTableName = "commits" 12 | // BlobsTableName is the name of the blobs table. 13 | BlobsTableName = "blobs" 14 | // TreeEntriesTableName is the name of the tree entries table. 15 | TreeEntriesTableName = "tree_entries" 16 | // RepositoriesTableName is the name of the repositories table. 17 | RepositoriesTableName = "repositories" 18 | // RemotesTableName is the name of the remotes table. 19 | RemotesTableName = "remotes" 20 | // RefCommitsTableName is the name of the ref commits table. 21 | RefCommitsTableName = "ref_commits" 22 | // CommitTreesTableName is the name of the commit trees table. 23 | CommitTreesTableName = "commit_trees" 24 | // CommitBlobsTableName is the name of the commit blobs table. 25 | CommitBlobsTableName = "commit_blobs" 26 | // CommitFilesTableName is the name of the commit files table. 27 | CommitFilesTableName = "commit_files" 28 | // FilesTableName is the name of the files table. 29 | FilesTableName = "files" 30 | ) 31 | 32 | // Database holds all git repository tables 33 | type Database struct { 34 | name string 35 | commits sql.Table 36 | references sql.Table 37 | treeEntries sql.Table 38 | blobs sql.Table 39 | repositories sql.Table 40 | remotes sql.Table 41 | refCommits sql.Table 42 | commitTrees sql.Table 43 | commitBlobs sql.Table 44 | commitFiles sql.Table 45 | files sql.Table 46 | } 47 | 48 | // NewDatabase creates a new Database structure and initializes its 49 | // tables with the given pool 50 | func NewDatabase(name string, pool *RepositoryPool) sql.Database { 51 | return &Database{ 52 | name: name, 53 | commits: newCommitsTable(pool), 54 | references: newReferencesTable(pool), 55 | blobs: newBlobsTable(pool), 56 | treeEntries: newTreeEntriesTable(pool), 57 | repositories: newRepositoriesTable(pool), 58 | remotes: newRemotesTable(pool), 59 | refCommits: newRefCommitsTable(pool), 60 | commitTrees: newCommitTreesTable(pool), 61 | commitBlobs: newCommitBlobsTable(pool), 62 | commitFiles: newCommitFilesTable(pool), 63 | files: newFilesTable(pool), 64 | } 65 | } 66 | 67 | // Name returns the name of the database 68 | func (d *Database) Name() string { 69 | return d.name 70 | } 71 | 72 | // Tables returns a map with all initialized tables 73 | func (d *Database) Tables() map[string]sql.Table { 74 | return map[string]sql.Table{ 75 | CommitsTableName: d.commits, 76 | ReferencesTableName: d.references, 77 | BlobsTableName: d.blobs, 78 | TreeEntriesTableName: d.treeEntries, 79 | RepositoriesTableName: d.repositories, 80 | RemotesTableName: d.remotes, 81 | RefCommitsTableName: d.refCommits, 82 | CommitTreesTableName: d.commitTrees, 83 | CommitBlobsTableName: d.commitBlobs, 84 | CommitFilesTableName: d.commitFiles, 85 | FilesTableName: d.files, 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /database_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | 7 | "github.com/src-d/go-borges/libraries" 8 | "github.com/src-d/go-mysql-server/sql" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | const testDBName = "foo" 13 | 14 | func TestDatabase_Tables(t *testing.T) { 15 | require := require.New(t) 16 | 17 | lib := libraries.New(nil) 18 | db := getDB(t, testDBName, NewRepositoryPool(nil, lib)) 19 | 20 | tables := db.Tables() 21 | var tableNames []string 22 | for key := range tables { 23 | tableNames = append(tableNames, key) 24 | } 25 | 26 | sort.Strings(tableNames) 27 | expected := []string{ 28 | CommitsTableName, 29 | CommitTreesTableName, 30 | RefCommitsTableName, 31 | ReferencesTableName, 32 | TreeEntriesTableName, 33 | BlobsTableName, 34 | RepositoriesTableName, 35 | RemotesTableName, 36 | CommitBlobsTableName, 37 | FilesTableName, 38 | CommitFilesTableName, 39 | } 40 | sort.Strings(expected) 41 | 42 | require.Equal(expected, tableNames) 43 | } 44 | 45 | func TestDatabase_Name(t *testing.T) { 46 | require := require.New(t) 47 | 48 | lib := libraries.New(nil) 49 | db := getDB(t, testDBName, NewRepositoryPool(nil, lib)) 50 | require.Equal(testDBName, db.Name()) 51 | } 52 | 53 | func getDB(t *testing.T, name string, pool *RepositoryPool) sql.Database { 54 | t.Helper() 55 | db := NewDatabase(name, pool) 56 | require.NotNil(t, db) 57 | 58 | return db 59 | } 60 | 61 | func getTable(t *testing.T, name string, ctx *sql.Context) sql.Table { 62 | t.Helper() 63 | require := require.New(t) 64 | db := getDB(t, "foo", poolFromCtx(t, ctx)) 65 | require.NotNil(db) 66 | require.Equal(db.Name(), "foo") 67 | 68 | tables := db.Tables() 69 | table, ok := tables[name] 70 | require.True(ok, "table %s does not exist", table) 71 | require.NotNil(table) 72 | 73 | return table 74 | } 75 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # gitbase 2 | 3 | * [Join the community](join-the-community.md) 4 | 5 | ### Using gitbase 6 | 7 | * [Getting started](using-gitbase/getting-started.md) 8 | * [Configuration](using-gitbase/configuration.md) 9 | * [Security](using-gitbase/security.md) 10 | * [Schema](using-gitbase/schema.md) 11 | * [Supported syntax](using-gitbase/supported-syntax.md) 12 | * [Supported clients](using-gitbase/supported-clients.md) 13 | * [Supported languages](using-gitbase/supported-languages.md) 14 | * [Functions](using-gitbase/functions.md) 15 | * [Indexes](using-gitbase/indexes.md) 16 | * [Examples](using-gitbase/examples.md) 17 | * [Optimizing queries](using-gitbase/optimize-queries.md) 18 | -------------------------------------------------------------------------------- /docs/assets/gitbase-schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/src-d/gitbase/c9a16f3b6d8171b595570787fbc2170325c08b48/docs/assets/gitbase-schema.png -------------------------------------------------------------------------------- /docs/assets/gitbase_model.mwb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/src-d/gitbase/c9a16f3b6d8171b595570787fbc2170325c08b48/docs/assets/gitbase_model.mwb -------------------------------------------------------------------------------- /docs/join-the-community.md: -------------------------------------------------------------------------------- 1 | # Join the community 2 | 3 | ## Chat 4 | 5 | If you need support, want to contribute or just want to say hi, join us at the [source{d} community Slack](https://join.slack.com/t/sourced-community/shared_invite/enQtMjc4Njk5MzEyNzM2LTFjNzY4NjEwZGEwMzRiNTM4MzRlMzQ4MmIzZjkwZmZlM2NjODUxZmJjNDI1OTcxNDAyMmZlNmFjODZlNTg0YWM). We hang out in the #general channel. 6 | 7 | ## Contributing 8 | 9 | You can start contributing in many ways: 10 | 11 | * [Report bugs](/docs/join-the-community.md#reporting-bugs) 12 | * [Request a feature](https://github.com/src-d/gitbase/issues) 13 | * Improve the [documentation](https://github.com/src-d/gitbase/tree/master/docs) 14 | * Contribute code to [gitbase](https://github.com/src-d/gitbase) 15 | 16 | ## Reporting bugs 17 | 18 | Bugs should be reported through [GitHub Issues](https://github.com/src-d/gitbase/issues). 19 | -------------------------------------------------------------------------------- /docs/using-gitbase/getting-started.md: -------------------------------------------------------------------------------- 1 | # Getting started 2 | 3 | ## Prerequisites 4 | 5 | `gitbase` optional dependencies that should be running on your system if you're planning on using certain functionality. 6 | 7 | - [bblfsh](https://github.com/bblfsh/bblfshd) >= 2.14.0 (only if you're planning to use the `UAST` functionality provided in gitbase) 8 | 9 | ## Installing gitbase 10 | 11 | The easiest way to run the gitbase server is using Docker; However, you have the options of using the binary or installing from source. 12 | 13 | ### Running with Docker 14 | 15 | You can use the official image from [docker hub](https://hub.docker.com/r/srcd/gitbase/tags/) to quickly run gitbase: 16 | ``` 17 | docker run --rm --name gitbase -p 3306:3306 -v /my/git/repos:/opt/repos srcd/gitbase:latest 18 | ``` 19 | 20 | **Note:** remember to replace `/my/git/repos` with the local path where your repositories are stored in your computer. 21 | 22 | If you want to use [bblfsh](https://github.com/bblfsh/bblfshd) with running in Docker you can do so by linking the 2 containers. 23 | Fist you need to start following [the bblfsh quick start](https://github.com/bblfsh/bblfshd#quick-start). After that you can run gitbase using: 24 | ``` 25 | docker run --rm --name gitbase -p 3306:3306 --link bblfshd:bblfshd -e BBLFSH_ENDPOINT=bblfshd:9432 -v /my/git/repos/go:/opt/repos srcd/gitbase:latest 26 | ``` 27 | 28 | ### Download and use the binary 29 | 30 | Check the [Releases](https://github.com/src-d/gitbase/releases) page to download the gitbase binary. 31 | 32 | For more info about command line arguments, [go here](/docs/using-gitbase/configuration.md#command-line-arguments). 33 | 34 | You can start a server by providing a path which contains multiple git repositories with this command: 35 | 36 | ``` 37 | gitbase server -v -d /path/to/repositories 38 | ``` 39 | 40 | **Note:** remember to replace `/path/to/repositories` with the local path where your repositories are stored in your computer. 41 | 42 | ### Installing from source 43 | 44 | On Linux and macOS: 45 | 46 | ``` 47 | go get -u github.com/src-d/gitbase/... 48 | ``` 49 | 50 | #### Oniguruma support 51 | 52 | On linux and macOS you can choose to build gitbase with oniguruma support, resulting in faster results for queries using the `language` UDF. 53 | 54 | macOS: 55 | 56 | ``` 57 | brew install oniguruma 58 | ``` 59 | 60 | Linux: 61 | 62 | - Debian-based distros: 63 | ``` 64 | sudo apt-get install libonig2 libonig-dev 65 | ``` 66 | - Arch linux: 67 | ``` 68 | pacman -S oniguruma 69 | ``` 70 | 71 | Then build gitbase like this: 72 | 73 | ``` 74 | go build -tags oniguruma -o gitbase ./cmd/gitbase/main.go 75 | ``` 76 | 77 | **Note:** prebuilt binaries do not include oniguruma support. 78 | 79 | On Windows: 80 | 81 | Because gitbase uses [bblfsh's client-go](https://github.com/bblfsh/client-go), which uses cgo, you need to install some dependencies by hand instead of just using `go get`. Use this instead: 82 | 83 | ``` 84 | go get -d github.com/src-d/gitbase 85 | cd $GOPATH/src/github.com/src-d/gitbase 86 | make dependencies 87 | ``` 88 | 89 | ## Connecting to the server 90 | 91 | When the gitbase server is started a MySQL client is needed to connect to the server. For example: 92 | 93 | ```bash 94 | $ mysql -q -u root -h 127.0.0.1 95 | MySQL [(none)]> SELECT commit_hash, commit_author_email, commit_author_name FROM commits LIMIT 2; 96 | SELECT commit_hash, commit_author_email, commit_author_name FROM commits LIMIT 2; 97 | +------------------------------------------+---------------------+-----------------------+ 98 | | commit_hash | commit_author_email | commit_author_name | 99 | +------------------------------------------+---------------------+-----------------------+ 100 | | 003dc36e0067b25333cb5d3a5ccc31fd028a1c83 | user1@test.io | Santiago M. Mola | 101 | | 01ace9e4d144aaeb50eb630fed993375609bcf55 | user2@test.io | Antonio Navarro Perez | 102 | +------------------------------------------+---------------------+-----------------------+ 103 | 2 rows in set (0.01 sec) 104 | ``` 105 | 106 | If you're using a MySQL client version 8.0 or higher, see the following section to solve some problems you may encounter. 107 | 108 | ## Troubleshooting 109 | 110 | ``` 111 | ERROR 2012 (HY000): Client asked for auth caching_sha2_password, but server wants auth mysql_native_password 112 | ``` 113 | 114 | As of MySQL 8.0 [the default authentication method is `caching_sha2_password`](https://dev.mysql.com/doc/refman/8.0/en/caching-sha2-pluggable-authentication.html) instead of `mysql_native_password`. You can solve this using the following command instead: 115 | 116 | ``` 117 | mysql -q -u root -h 127.0.0.1 --default-auth=mysql_native_password 118 | ``` 119 | 120 | ## Library format specification 121 | 122 | By default the directories added to gitbase should contain git repositories and it detects if they are standard or bare format. Each directory added can only contain one type of repository. If you want to specify the format you have two ways to do it: 123 | 124 | If all the directories are in the same format you can set it globally with these parameters: 125 | 126 | * `--format`: it can be either `git` for filesystem repositories or `siva` for siva archives 127 | * `--bare`: specifies that git archives are bare, can only be used with `git` format 128 | * `--non-bare`: specifies that git archives are standard, can only be used with `git` format 129 | * `--bucket`: sets the number of characters to use for bucketing, used with `siva` libraries 130 | * `--non-rooted`: disables rooted repositories management in `siva` libraries 131 | 132 | If you are mixing formats you can specify each directory as a `file://` URL with these parameters: 133 | 134 | * `format`: can be `git` or `siva` 135 | * `bare`: `true`, `false` or `auto` 136 | * `bucket`: the characters to use for directory bucketing 137 | * `rooted`: `true` or `false` 138 | 139 | For example: 140 | 141 | ``` 142 | -d 'file:///path/to/git?format=git&bare=true' -d 'file:///path/to/sivas?format=siva&rooted=false&bucket=0' 143 | ``` 144 | -------------------------------------------------------------------------------- /docs/using-gitbase/indexes.md: -------------------------------------------------------------------------------- 1 | # Indexes 2 | 3 | `gitbase` allows you to speed up queries creating indexes. 4 | 5 | Indexes are implemented as bitmaps using [pilosa](https://github.com/pilosa/pilosa) as a backend storage for them. 6 | 7 | Thus, to create indexes you must specify pilosa as the type of index. You can find some examples in the [examples](./examples.md#create-an-index-for-columns-on-a-table) section about managing indexes. 8 | 9 | Note that you can create an index either **on one or more columns** or **on a single expression**. 10 | In practice, having multiple indexes (one per column) is better and more flexible than one index for multiple columns. It is because of data structures (bitmaps) used to represent index values. 11 | Even if you have one index on multiple columns, every column is stored in an independent _field_. 12 | Merging those _fields_ by any logic operations is fast and much more flexible. The main difference of having multiple columns per index is, it internally calculates intersection across columns, so the index won't be used if you use _non_ `AND` operation in a filter, e.g.: 13 | 14 | With index on (`A`, `B`), the index will be used for following query: 15 | ```sql 16 | SELECT * FROM T WHERE A='...' AND B='...' 17 | ``` 18 | But it won't be used if we change logic operation to, e.g.: 19 | ```sql 20 | SELECT * FROM T WHERE A='...' OR B='...' 21 | ``` 22 | 23 | So it's more flexible with two indexes - one on `A`, and the second on `B`. 24 | For the first query the intersection of two _fields_ will be returned 25 | and for the second query also two indexes will be used and the result will be a union. 26 | 27 | You can find some more examples in the [examples](./examples.md#create-an-index-for-columns-on-a-table) section. 28 | 29 | See [go-mysql-server](https://github.com/src-d/go-mysql-server/tree/541fde3b92093b3a449e803342a7a18c686275e6#indexes) documentation for more details. 30 | -------------------------------------------------------------------------------- /docs/using-gitbase/schema.md: -------------------------------------------------------------------------------- 1 | # Schema 2 | 3 | You can execute the `SHOW TABLES` statement to get a list of the available tables. 4 | To get all the columns and types of a specific table, you can write `DESCRIBE TABLE [tablename]`. 5 | 6 | gitbase exposes the following tables: 7 | 8 | ## Main tables 9 | 10 | ### repositories 11 | ``` sql 12 | +---------------+------+ 13 | | name | type | 14 | +---------------+------+ 15 | | repository_id | TEXT | 16 | +---------------+------+ 17 | ``` 18 | 19 | Table that contains all the repositories on the dataset. `repository_id` is the path to the repository folder. 20 | 21 | In case of [siva files](https://github.com/src-d/go-siva/), the id is the path + the siva file name. 22 | 23 | ### remotes 24 | ``` sql 25 | +----------------------+------+ 26 | | name | type | 27 | +----------------------+------+ 28 | | repository_id | TEXT | 29 | | remote_name | TEXT | 30 | | remote_push_url | TEXT | 31 | | remote_fetch_url | TEXT | 32 | | remote_push_refspec | TEXT | 33 | | remote_fetch_refspec | TEXT | 34 | +----------------------+------+ 35 | ``` 36 | 37 | This table will return all the [remotes](https://git-scm.com/book/en/v2/Git-Basics-Working-with-Remotes) configured on git `config` file of all the repositories. 38 | 39 | ### refs 40 | ``` sql 41 | +---------------+-------------+ 42 | | name | type | 43 | +---------------+-------------+ 44 | | repository_id | TEXT | 45 | | ref_name | TEXT | 46 | | commit_hash | VARCHAR(40) | 47 | +---------------+-------------+ 48 | ``` 49 | This table contains all hash [git references](https://git-scm.com/book/en/v2/Git-Internals-Git-References) and the symbolic reference `HEAD` from all the repositories. 50 | 51 | ### commits 52 | ``` sql 53 | +---------------------+--------------+ 54 | | name | type | 55 | +---------------------+--------------+ 56 | | repository_id | TEXT | 57 | | commit_hash | VARCHAR(40) | 58 | | commit_author_name | TEXT | 59 | | commit_author_email | VARCHAR(254) | 60 | | commit_author_when | TIMESTAMP | 61 | | committer_name | TEXT | 62 | | committer_email | VARCHAR(254) | 63 | | committer_when | TIMESTAMP | 64 | | commit_message | TEXT | 65 | | tree_hash | TEXT | 66 | | commit_parents | JSON | 67 | +---------------------+--------------+ 68 | ``` 69 | 70 | Commits contains all the [commits](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects#_git_commit_objects) from all the references from all the repositories, not duplicated by repository. Note that you can have the same commit in several repositories. In that case the commit will appear two times on the table, one per repository. 71 | 72 | > Note that this table is not only showing `HEAD` commits but all the commits on the repository (that can be a lot more than the commits on `HEAD` reference). 73 | 74 | ### blobs 75 | ```sql 76 | +---------------+--------------+ 77 | | name | type | 78 | +---------------+--------------+ 79 | | repository_id | TEXT | 80 | | blob_hash | VARCHAR(40) | 81 | | blob_size | INT64 | 82 | | blob_content | BLOB | 83 | +---------------+--------------+ 84 | ``` 85 | 86 | This table exposes blob objects, that are the content without path from files. 87 | 88 | > Note that this table will return all the existing blobs on all the commits on all the repositories, potentially **a lot** of data. In most common cases you want to filter by commit, by reference or by repository. 89 | 90 | ### tree_entries 91 | ```sql 92 | +-----------------+-------------+ 93 | | name | type | 94 | +-----------------+-------------+ 95 | | repository_id | TEXT | 96 | | tree_entry_name | TEXT | 97 | | blob_hash | VARCHAR(40) | 98 | | tree_hash | VARCHAR(40) | 99 | | tree_entry_mode | VARCHAR(16) | 100 | +-----------------+-------------+ 101 | ``` 102 | 103 | `tree_entries` table contains all the objects from all the repositories that are [tree objects](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects#_git_commit_objects). 104 | 105 | 106 | ### files 107 | ```sql 108 | +-----------------+--------------+ 109 | | name | type | 110 | +-----------------+--------------+ 111 | | repository_id | TEXT | 112 | | file_path | TEXT | 113 | | blob_hash | VARCHAR(40) | 114 | | tree_hash | VARCHAR(40) | 115 | | tree_entry_mode | VARCHAR(16) | 116 | | blob_content | BLOB | 117 | | blob_size | INT64 | 118 | +-----------------+--------------+ 119 | ``` 120 | 121 | `files` is an utility table mixing `tree_entries` and `blobs` to create files. It includes the file path. 122 | 123 | Queries to this table are expensive and they should be done carefully (applying filters or using directly `blobs` or `tree_entries` tables). 124 | 125 | ## Relation tables 126 | 127 | ### commit_blobs 128 | ```sql 129 | +---------------+-------------+ 130 | | name | type | 131 | +---------------+-------------+ 132 | | repository_id | TEXT | 133 | | commit_hash | VARCHAR(40) | 134 | | blob_hash | VARCHAR(40) | 135 | +---------------+-------------+ 136 | ``` 137 | 138 | This table represents the relation between commits and blobs. With this table you can obtain all the blobs contained on a commit object. 139 | 140 | ### commit_trees 141 | ```sql 142 | +---------------+-------------+ 143 | | name | type | 144 | +---------------+-------------+ 145 | | repository_id | TEXT | 146 | | commit_hash | VARCHAR(40) | 147 | | tree_hash | VARCHAR(40) | 148 | +---------------+-------------+ 149 | ``` 150 | 151 | This table represents the relation between commits and trees. With this table you can obtain all the tree entries contained on a commit object. 152 | 153 | ### commit_files 154 | ```sql 155 | +----------------------+------+ 156 | | name | type | 157 | +----------------------+------+ 158 | | repository_id | TEXT | 159 | | commit_hash | VARCHAR(40) | 160 | | file_path | TEXT | 161 | | blob_hash | VARCHAR(40) | 162 | | tree_hash | VARCHAR(40) | 163 | +---------------+-------------+ 164 | ``` 165 | 166 | This table represents the relation between commits and [files](#files). Using this table, you can obtain all the files related to a certain commit object. 167 | 168 | ### ref_commits 169 | ```sql 170 | +---------------+--------------+ 171 | | name | type | 172 | +---------------+--------------+ 173 | | repository_id | TEXT | 174 | | commit_hash | VARCHAR(40) | 175 | | ref_name | TEXT | 176 | | history_index | INT64 | 177 | +---------------+--------------+ 178 | ``` 179 | 180 | This table allow us to get the commit history from a specific reference name. `history_index` column represents the position of the commit from a specific reference. 181 | 182 | This table is like the [log](https://git-scm.com/docs/git-log) from a specific reference. 183 | 184 | Commits will be repeated if they are in several repositories or references. 185 | 186 | ## Database diagram 187 | 192 | 193 | ![gitbase schema](/docs/assets/gitbase-schema.png) 194 | -------------------------------------------------------------------------------- /docs/using-gitbase/security.md: -------------------------------------------------------------------------------- 1 | # Security 2 | 3 | ## User credentials 4 | 5 | User credentials can be specified in the command line or in the user file. For a single user this can be done with parameters `--user` and `--password`: 6 | 7 | ``` 8 | gitbase server --user root --password r00tp4ssword! -d /my/repositories/path 9 | ``` 10 | 11 | If you want to have more than one user or not having the password in plain text, then the user file can be useful. You can keep passwords, permissions and user names in the following format: 12 | 13 | ```json 14 | [ 15 | { 16 | "name": "root", 17 | "password": "*2470C0C06DEE42FD1618BB99005ADCA2EC9D1E19", 18 | "permissions": ["read", "write"] 19 | }, 20 | { 21 | "name": "user", 22 | "password": "plain_passw0rd!" 23 | } 24 | ] 25 | ``` 26 | 27 | You can either specify passwords as a plain text or hashed. Hashed version uses the same format as MySQL 5.x passwords. You can generate the native password with this command, remember to prefix the hash with `*`: 28 | 29 | ``` 30 | echo -n password | openssl sha1 -binary | openssl sha1 | tr '[:lower:]' '[:upper:]' 31 | ``` 32 | 33 | There are two permissions you can set for users, `read` and `write`. `read` only allows executing read-only queries that do not modify the internal state or the data itself. `write` is needed to create and delete indexes or lock tables. If no permissions are set for a user the default permission is `read`. 34 | 35 | Then you can specify which user file to use with the `--user-file` parameter: 36 | 37 | ``` 38 | gitbase server --user-file /path/to/user-file.json -d /my/repositories/path 39 | ``` 40 | 41 | ## Audit 42 | 43 | Gitbase offers audit trails on logs. Right now, we have three different kinds of records: `authentication`, `authorization` and `query` 44 | 45 | ### Authentication 46 | 47 | Record triggered when a user is trying to connect to gitbase. It contains the following information: 48 | 49 | - action: Always `authentication`. 50 | - system: Always `audit` 51 | - address: Address of the client trying to connect. 52 | - err: Human readable error if the authentication was not successful. 53 | - success: True or false depending on whether the client authenticated correctly or not. 54 | - user: Username trying to connect 55 | 56 | Example: 57 | 58 | ``` 59 | action=authentication address="127.0.0.1:41720" err="Access denied for user 'test' (errno 1045) (sqlstate 28000)" success=false system=audit user=test 60 | ``` 61 | 62 | ### Authorization 63 | 64 | Record triggered checking when a user is authorized to execute a specific valid query with their permissions. It contains the following information: 65 | 66 | - action: Always `authorization`. 67 | - system: Always `audit` 68 | - address: Address of the client. 69 | - success: True or false depending on whether the client has been authorized correctly or not. 70 | - user: Username trying to execute the query. 71 | - connection_id: Unique connection identifier of the current request. 72 | - permission: Permission needed to execute the query. 73 | - pid: Pid returns the process ID associated with this context. It will change in subsequent queries sent using the same connection. 74 | - query: Query that client is trying to execute. 75 | 76 | Example: 77 | 78 | ``` 79 | INFO[0007] audit trail action=authorization address="127.0.0.1:41610" connection_id=1 permission=read pid=1 query="select @@version_comment limit 1" success=true system=audit user=root 80 | ``` 81 | 82 | ### Query 83 | 84 | Record triggered at the end of the executed query. It contains the following information: 85 | 86 | - action: Always `query`. 87 | - system: Always `audit` 88 | - address: Address of the client. 89 | - success: True or false depending on whether the query was executed or not. 90 | - user: Username executing the query. 91 | - connection_id: Unique connection identifier of the current request. 92 | - pid: Pid returns the process ID associated with this context. It will change in subsequent queries sent using the same connection. 93 | - query: Query that client is trying to execute. 94 | - err: If `success=false`. Human readable error describing the problem. 95 | - duration: Time that the query took to execute. If the format of the logs is `JSON` this duration unit is nanoseconds. 96 | 97 | Examples: 98 | 99 | ``` 100 | INFO[0983] audit trail action=query address="127.0.0.1:42428" connection_id=2 duration=22.707457818s pid=6 query="select count(*) from commits" success=true system=audit user=root 101 | ``` 102 | 103 | ``` 104 | INFO[0910] audit trail action=query address="127.0.0.1:42428" connection_id=2 duration="77.822µs" err="syntax error at position 6 near 'wrong'" pid=5 query="wrong query" success=false system=audit user=root 105 | ``` -------------------------------------------------------------------------------- /docs/using-gitbase/supported-clients.md: -------------------------------------------------------------------------------- 1 | # Supported clients 2 | 3 | These are the clients we actively test against to check that they are compatible with go-mysql-server. Other clients may also work, but we don't check on every build if we remain compatible with them. 4 | 5 | - Python 6 | - [pymysql](#pymysql) 7 | - [mysql-connector](#python-mysql-connector) 8 | - [sqlalchemy](#python-sqlalchemy) 9 | - Ruby 10 | - [ruby-mysql](#ruby-mysql) 11 | - [PHP](#php) 12 | - Node.js 13 | - [mysqljs/mysql](#mysqljs) 14 | - .NET Core 15 | - [MysqlConnector](#mysqlconnector) 16 | - Java/JVM 17 | - [mariadb-java-client](#mariadb-java-client) 18 | - Go 19 | - [go-mysql-driver/mysql](#go-sql-drivermysql) 20 | - C 21 | - [mysql-connector-c](#mysql-connector-c) 22 | - Grafana 23 | - Tableau Desktop 24 | 25 | ## Example client usage 26 | 27 | ### pymysql 28 | 29 | ```python 30 | import pymysql.cursors 31 | 32 | connection = pymysql.connect(host='127.0.0.1', 33 | user='root', 34 | password='', 35 | db='mydb', 36 | cursorclass=pymysql.cursors.DictCursor) 37 | 38 | try: 39 | with connection.cursor() as cursor: 40 | sql = "SELECT * FROM mytable LIMIT 1" 41 | cursor.execute(sql) 42 | rows = cursor.fetchall() 43 | 44 | # use rows 45 | finally: 46 | connection.close() 47 | ``` 48 | 49 | ### Python mysql-connector 50 | 51 | ```python 52 | import mysql.connector 53 | 54 | connection = mysql.connector.connect(host='127.0.0.1', 55 | user='root', 56 | passwd='', 57 | port=3306, 58 | database='mydb') 59 | 60 | try: 61 | cursor = connection.cursor() 62 | sql = "SELECT * FROM mytable LIMIT 1" 63 | cursor.execute(sql) 64 | rows = cursor.fetchall() 65 | 66 | # use rows 67 | finally: 68 | connection.close() 69 | ``` 70 | 71 | ### Python sqlalchemy 72 | 73 | ```python 74 | import pandas as pd 75 | import sqlalchemy 76 | 77 | engine = sqlalchemy.create_engine('mysql+pymysql://root:@127.0.0.1:3306/mydb') 78 | with engine.connect() as conn: 79 | repo_df = pd.read_sql_table("mytable", con=conn) 80 | for table_name in repo_df.to_dict(): 81 | print(table_name) 82 | ``` 83 | 84 | ### ruby-mysql 85 | 86 | ```ruby 87 | require "mysql" 88 | 89 | conn = Mysql::new("127.0.0.1", "root", "", "mydb") 90 | resp = conn.query "SELECT * FROM mytable LIMIT 1" 91 | 92 | # use resp 93 | 94 | conn.close() 95 | ``` 96 | 97 | ### php 98 | 99 | ```php 100 | try { 101 | $conn = new PDO("mysql:host=127.0.0.1:3306;dbname=mydb", "root", ""); 102 | $conn->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); 103 | 104 | $stmt = $conn->query('SELECT * FROM mytable LIMIT 1'); 105 | $result = $stmt->fetchAll(PDO::FETCH_ASSOC); 106 | 107 | // use result 108 | } catch (PDOException $e) { 109 | // handle error 110 | } 111 | ``` 112 | 113 | ### mysqljs 114 | 115 | ```js 116 | import mysql from 'mysql'; 117 | 118 | const connection = mysql.createConnection({ 119 | host: '127.0.0.1', 120 | port: 3306, 121 | user: 'root', 122 | password: '', 123 | database: 'mydb' 124 | }); 125 | connection.connect(); 126 | 127 | const query = 'SELECT * FROM mytable LIMIT 1'; 128 | connection.query(query, function (error, results, _) { 129 | if (error) throw error; 130 | 131 | // use results 132 | }); 133 | 134 | connection.end(); 135 | ``` 136 | 137 | ### MysqlConnector 138 | 139 | ```csharp 140 | using MySql.Data.MySqlClient; 141 | using System.Threading.Tasks; 142 | 143 | namespace something 144 | { 145 | public class Something 146 | { 147 | public async Task DoQuery() 148 | { 149 | var connectionString = "server=127.0.0.1;user id=root;password=;port=3306;database=mydb;"; 150 | 151 | using (var conn = new MySqlConnection(connectionString)) 152 | { 153 | await conn.OpenAsync(); 154 | 155 | var sql = "SELECT * FROM mytable LIMIT 1"; 156 | 157 | using (var cmd = new MySqlCommand(sql, conn)) 158 | using (var reader = await cmd.ExecuteReaderAsync()) 159 | while (await reader.ReadAsync()) { 160 | // use reader 161 | } 162 | } 163 | } 164 | } 165 | } 166 | ``` 167 | 168 | ### mariadb-java-client 169 | 170 | ```java 171 | package org.testing.mariadbjavaclient; 172 | 173 | import java.sql.*; 174 | 175 | class Main { 176 | public static void main(String[] args) { 177 | String dbUrl = "jdbc:mariadb://127.0.0.1:3306/mydb?user=root&password="; 178 | String query = "SELECT * FROM mytable LIMIT 1"; 179 | 180 | try (Connection connection = DriverManager.getConnection(dbUrl)) { 181 | try (PreparedStatement stmt = connection.prepareStatement(query)) { 182 | try (ResultSet rs = stmt.executeQuery()) { 183 | while (rs.next()) { 184 | // use rs 185 | } 186 | } 187 | } 188 | } catch (SQLException e) { 189 | // handle failure 190 | } 191 | } 192 | } 193 | ``` 194 | 195 | ### go-sql-driver/mysql 196 | 197 | ```go 198 | package main 199 | 200 | import ( 201 | "database/sql" 202 | 203 | _ "github.com/go-sql-driver/mysql" 204 | ) 205 | 206 | func main() { 207 | db, err := sql.Open("mysql", "root:@tcp(127.0.0.1:3306)/mydb") 208 | if err != nil { 209 | // handle error 210 | } 211 | 212 | rows, err := db.Query("SELECT * FROM mytable LIMIT 1") 213 | if err != nil { 214 | // handle error 215 | } 216 | 217 | // use rows 218 | } 219 | ``` 220 | 221 | ### mysql-connector-c 222 | 223 | ```c 224 | #include 225 | #include 226 | 227 | void finish_with_error(MYSQL *con) 228 | { 229 | fprintf(stderr, "%s\n", mysql_error(con)); 230 | mysql_close(con); 231 | exit(1); 232 | } 233 | 234 | int main(int argc, char **argv) 235 | { 236 | MYSQL *con = NULL; 237 | MYSQL_RES *result = NULL; 238 | int num_fields = 0; 239 | MYSQL_ROW row; 240 | 241 | printf("MySQL client version: %s\n", mysql_get_client_info()); 242 | 243 | con = mysql_init(NULL); 244 | if (con == NULL) { 245 | finish_with_error(con); 246 | } 247 | 248 | if (mysql_real_connect(con, "127.0.0.1", "root", "", "mydb", 3306, NULL, 0) == NULL) { 249 | finish_with_error(con); 250 | } 251 | 252 | if (mysql_query(con, "SELECT name, email, phone_numbers FROM mytable")) { 253 | finish_with_error(con); 254 | } 255 | 256 | result = mysql_store_result(con); 257 | if (result == NULL) { 258 | finish_with_error(con); 259 | } 260 | 261 | num_fields = mysql_num_fields(result); 262 | while ((row = mysql_fetch_row(result))) { 263 | for(int i = 0; i < num_fields; i++) { 264 | printf("%s ", row[i] ? row[i] : "NULL"); 265 | } 266 | printf("\n"); 267 | } 268 | 269 | mysql_free_result(result); 270 | mysql_close(con); 271 | 272 | return 0; 273 | } 274 | ``` 275 | -------------------------------------------------------------------------------- /docs/using-gitbase/supported-languages.md: -------------------------------------------------------------------------------- 1 | # Supported languages 2 | 3 | Gitbase supports many programming languages depending on the use case. 4 | For instance the `language(path, [blob])` function supports all languages which [enry's package](https://github.com/src-d/enry) can autodetect. 5 | More details about aliases, groups, extensions, etc. you can find in [enry's repo](https://github.com/src-d/enry/blob/master/data/alias.go), 6 | or go directly to [linguist defines](https://github.com/github/linguist/blob/master/lib/linguist/languages.yml). 7 | 8 | If your use case requires _Universal Abstract Syntax Tree_ then most likely one of the following functions will be interesting for you: 9 | - `uast(blob, [lang, [xpath]])` 10 | - `uast_mode(mode, blob, lang)` 11 | - `uast_xpath(blob, xpath)` 12 | - `uast_extract(blob, key)` 13 | - `uast_children(blob)` 14 | 15 | The _UAST_ functions support programming languages which already have implemented [babelfish](https://docs.sourced.tech/babelfish) driver. 16 | The list of currently supported languages on babelfish, you can find [here](https://docs.sourced.tech/babelfish/languages#supported-languages). 17 | Drivers which are still in development can be found [here](https://docs.sourced.tech/babelfish/languages#in-development). 18 | -------------------------------------------------------------------------------- /docs/using-gitbase/supported-syntax.md: -------------------------------------------------------------------------------- 1 | # Supported SQL Syntax 2 | 3 | ## Comparisson expressions 4 | - != 5 | - == 6 | - \> 7 | - < 8 | - \>= 9 | - <= 10 | - BETWEEN 11 | - IN 12 | - NOT IN 13 | - REGEXP 14 | 15 | ## Null check expressions 16 | - IS NOT NULL 17 | - IS NULL 18 | 19 | ## Grouping expressions 20 | - AVG 21 | - COUNT and COUNT(DISTINCT) 22 | - MAX 23 | - MIN 24 | - SUM (always returns DOUBLE) 25 | 26 | ## Standard expressions 27 | - ALIAS (AS) 28 | - CAST/CONVERT 29 | - CREATE TABLE 30 | - DESCRIBE/DESC/EXPLAIN FORMAT=TREE [query] 31 | - DISTINCT 32 | - FILTER (WHERE) 33 | - GROUP BY 34 | - INSERT INTO 35 | - LIMIT/OFFSET 36 | - LITERAL 37 | - ORDER BY 38 | - SELECT 39 | - SHOW TABLES 40 | - SORT 41 | - STAR (*) 42 | - SHOW PROCESSLIST 43 | - SHOW TABLE STATUS 44 | - SHOW VARIABLES 45 | - SHOW CREATE DATABASE 46 | - SHOW CREATE TABLE 47 | - SHOW FIELDS FROM 48 | - LOCK/UNLOCK 49 | - USE 50 | - SHOW DATABASES 51 | - SHOW WARNINGS 52 | - INTERVALS 53 | 54 | ## Index expressions 55 | - CREATE INDEX (an index can be created using either column names or a single arbitrary expression). 56 | - DROP INDEX 57 | - SHOW {INDEXES | INDEX | KEYS} {FROM | IN} [table name] 58 | 59 | ## Join expressions 60 | - CROSS JOIN 61 | - INNER JOIN 62 | - NATURAL JOIN 63 | 64 | ## Logical expressions 65 | - AND 66 | - NOT 67 | - OR 68 | 69 | ## Arithmetic expressions 70 | - \+ (including between dates and intervals) 71 | - \- (including between dates and intervals) 72 | - \* 73 | - \\ 74 | - << 75 | - \>> 76 | - & 77 | - \| 78 | - ^ 79 | - div 80 | - % 81 | 82 | ## Functions 83 | - ARRAY_LENGTH 84 | - CEIL 85 | - CEILING 86 | - COALESCE 87 | - CONCAT 88 | - CONCAT_WS 89 | - CONNECTION_ID 90 | - DATABASE 91 | - FLOOR 92 | - FROM_BASE64 93 | - GREATEST 94 | - IS_BINARY 95 | - IS_BINARY 96 | - JSON_EXTRACT 97 | - JSON_UNQUOTE 98 | - LEAST 99 | - LN 100 | - LOG10 101 | - LOG2 102 | - LOWER 103 | - LPAD 104 | - POW 105 | - POWER 106 | - ROUND 107 | - RPAD 108 | - SLEEP 109 | - SOUNDEX 110 | - SPLIT 111 | - SQRT 112 | - SUBSTRING 113 | - TO_BASE64 114 | - UPPER 115 | 116 | ## Time functions 117 | - DATE 118 | - DATE_ADD 119 | - DATE_SUB 120 | - DAY 121 | - DAYOFMONTH 122 | - DAYOFWEEK 123 | - DAYOFYEAR 124 | - HOUR 125 | - MINUTE 126 | - MONTH 127 | - NOW 128 | - SECOND 129 | - WEEKDAY 130 | - YEAR 131 | - YEARWEEK 132 | 133 | ## Subqueries 134 | Supported both as a table and as expressions but they can't access the parent query scope. 135 | -------------------------------------------------------------------------------- /e2e/e2e_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "database/sql" 5 | "flag" 6 | "fmt" 7 | "net" 8 | "os" 9 | "os/exec" 10 | "testing" 11 | "time" 12 | 13 | _ "github.com/go-sql-driver/mysql" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | var ( 18 | mustRun = flag.Bool("must-run", false, "ensures the tests are run") 19 | bin = flag.String("gitbase-bin", "", "path to the gitbase binary to test") 20 | repos = flag.String("gitbase-repos", "", "path to the gitbase repos to test") 21 | version = flag.String("gitbase-version", "", "(optional) version of the binary") 22 | 23 | port int 24 | ) 25 | 26 | func TestMain(m *testing.M) { 27 | flag.Parse() 28 | 29 | if *repos == "" { 30 | fmt.Println("gitbase-repos not provided") 31 | if *mustRun { 32 | os.Exit(1) 33 | } else { 34 | os.Exit(0) 35 | } 36 | } 37 | 38 | path, err := exec.LookPath(*bin) 39 | if err != nil { 40 | fmt.Println("gitbase-bin not provided:", err) 41 | if *mustRun { 42 | os.Exit(1) 43 | } else { 44 | os.Exit(0) 45 | } 46 | } 47 | 48 | port, err = findPort() 49 | if err != nil { 50 | fmt.Println("unable to find an available port: ", err) 51 | os.Exit(1) 52 | } 53 | 54 | var done = make(chan error) 55 | cmd := exec.Command( 56 | path, 57 | "server", 58 | "--directories="+*repos, 59 | "--host=127.0.0.1", 60 | fmt.Sprintf("--port=%d", port), 61 | "--index=indexes", 62 | ) 63 | 64 | if err := cmd.Start(); err != nil { 65 | fmt.Println("unable to start gitbase binary:", err) 66 | os.Exit(1) 67 | } 68 | 69 | go func() { 70 | switch err := cmd.Wait().(type) { 71 | case *exec.ExitError: 72 | done <- nil 73 | default: 74 | done <- err 75 | } 76 | }() 77 | 78 | time.Sleep(500 * time.Millisecond) 79 | 80 | code := m.Run() 81 | 82 | if err := cmd.Process.Signal(os.Interrupt); err != nil { 83 | fmt.Println("problem stopping binary:", err) 84 | os.Exit(1) 85 | } 86 | 87 | if err := <-done; err != nil { 88 | fmt.Println("problem executing binary:", err) 89 | os.Exit(1) 90 | } 91 | 92 | os.Exit(code) 93 | } 94 | 95 | func connect(t *testing.T) (*sql.DB, func()) { 96 | db, err := sql.Open("mysql", fmt.Sprintf("root:@tcp(127.0.0.1:%d)/gitbase", port)) 97 | if err != nil { 98 | t.Errorf("unexpected error connecting to gitbase: %s", err) 99 | } 100 | 101 | return db, func() { 102 | require.NoError(t, db.Close()) 103 | } 104 | } 105 | 106 | func findPort() (int, error) { 107 | addr, err := net.ResolveTCPAddr("tcp", "localhost:0") 108 | if err != nil { 109 | return 0, err 110 | } 111 | 112 | l, err := net.ListenTCP("tcp", addr) 113 | if err != nil { 114 | return 0, err 115 | } 116 | 117 | port := l.Addr().(*net.TCPAddr).Port 118 | _ = l.Close() 119 | 120 | return port, nil 121 | } 122 | 123 | func TestVersion(t *testing.T) { 124 | if *version == "" { 125 | t.Skip("no version provided, skipping") 126 | } 127 | db, cleanup := connect(t) 128 | defer cleanup() 129 | 130 | require := require.New(t) 131 | 132 | var v string 133 | require.NoError(db.QueryRow("SELECT VERSION()").Scan(&v)) 134 | require.Equal(fmt.Sprintf("8.0.11-%s", *version), v) 135 | } 136 | -------------------------------------------------------------------------------- /env.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | ) 7 | 8 | func getIntEnv(key string, defaultValue int) int { 9 | val := os.Getenv(key) 10 | if val == "" { 11 | return defaultValue 12 | } 13 | v, err := strconv.Atoi(val) 14 | if err != nil { 15 | return defaultValue 16 | } 17 | return v 18 | } 19 | 20 | func getBoolEnv(key string, defaultValue bool) bool { 21 | _, ok := os.LookupEnv(key) 22 | if ok { 23 | return true 24 | } 25 | 26 | return defaultValue 27 | } 28 | 29 | func getStringEnv(key string, defaultValue string) string { 30 | v, ok := os.LookupEnv(key) 31 | if !ok { 32 | return defaultValue 33 | } 34 | return v 35 | } 36 | -------------------------------------------------------------------------------- /fs_error_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | "time" 11 | 12 | "github.com/sirupsen/logrus" 13 | "github.com/src-d/go-borges" 14 | "github.com/src-d/go-borges/plain" 15 | fixtures "github.com/src-d/go-git-fixtures" 16 | "github.com/src-d/go-mysql-server/sql" 17 | "github.com/stretchr/testify/require" 18 | billy "gopkg.in/src-d/go-billy.v4" 19 | "gopkg.in/src-d/go-billy.v4/osfs" 20 | ) 21 | 22 | func TestFSErrorTables(t *testing.T) { 23 | logrus.SetLevel(logrus.FatalLevel) 24 | 25 | tests := []struct { 26 | table string 27 | rows int 28 | }{ 29 | {BlobsTableName, 14}, 30 | {CommitBlobsTableName, 88}, 31 | {CommitFilesTableName, 88}, 32 | {CommitTreesTableName, 40}, 33 | {CommitsTableName, 9}, 34 | {FilesTableName, 82}, 35 | {RefCommitsTableName, 64}, 36 | {ReferencesTableName, 8}, 37 | {RepositoriesTableName, 3}, 38 | {TreeEntriesTableName, 45}, 39 | } 40 | 41 | for _, test := range tests { 42 | t.Run(test.table, func(t *testing.T) { 43 | testTable(t, test.table, test.rows) 44 | }) 45 | } 46 | } 47 | 48 | // setupErrorRepos creates a pool with three repos. One with read error in 49 | // packfile, another with an index file missing (and ghost packfile) and 50 | // finally a correct repository. 51 | func setupErrorRepos(t *testing.T) (*sql.Context, CleanupFunc) { 52 | require := require.New(t) 53 | 54 | t.Helper() 55 | 56 | fixture := fixtures.ByTag("worktree").One() 57 | baseFS := fixture.Worktree() 58 | tmpDir, err := ioutil.TempDir("", "gitbase") 59 | require.NoError(err) 60 | 61 | rootFS := osfs.New(tmpDir) 62 | 63 | lib, pool, err := newMultiPool() 64 | require.NoError(err) 65 | 66 | repos := []struct { 67 | name string 68 | t brokenType 69 | }{ 70 | { 71 | name: "packfile", 72 | t: brokenPackfile, 73 | }, 74 | { 75 | name: "index", 76 | t: brokenIndex, 77 | }, 78 | { 79 | name: "ok", 80 | t: brokenNone, 81 | }, 82 | } 83 | 84 | var fs billy.Filesystem 85 | for _, repo := range repos { 86 | err = rootFS.MkdirAll(repo.name, 0777) 87 | require.NoError(err) 88 | fs, err = rootFS.Chroot(repo.name) 89 | require.NoError(err) 90 | err = recursiveCopy(repo.name, fs, ".git", baseFS) 91 | require.NoError(err) 92 | fs, err = brokenFS(repo.t, fs) 93 | require.NoError(err) 94 | 95 | loc, err := plain.NewLocation(borges.LocationID(repo.name), fs, &plain.LocationOptions{ 96 | Bare: true, 97 | }) 98 | require.NoError(err) 99 | lib.plain.AddLocation(loc) 100 | } 101 | 102 | session := NewSession(pool, WithSkipGitErrors(true)) 103 | ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) 104 | 105 | cleanup := func() { 106 | t.Helper() 107 | require.NoError(fixtures.Clean()) 108 | require.NoError(os.RemoveAll(tmpDir)) 109 | } 110 | 111 | return ctx, cleanup 112 | } 113 | 114 | func brokenFS( 115 | brokenType brokenType, 116 | fs billy.Filesystem, 117 | ) (billy.Filesystem, error) { 118 | var brokenFS billy.Filesystem 119 | if brokenType == brokenNone { 120 | brokenFS = fs 121 | } else { 122 | brokenFS = NewBrokenFS(brokenType, fs) 123 | } 124 | 125 | return brokenFS, nil 126 | } 127 | 128 | func testTable(t *testing.T, tableName string, number int) { 129 | require := require.New(t) 130 | 131 | ctx, cleanup := setupErrorRepos(t) 132 | defer cleanup() 133 | 134 | table := getTable(t, tableName, ctx) 135 | rows, err := tableToRows(ctx, table) 136 | require.NoError(err) 137 | 138 | if len(rows) < number { 139 | t.Errorf("table %s returned %v rows and it should be at least %v", 140 | tableName, len(rows), number) 141 | t.FailNow() 142 | } 143 | } 144 | 145 | type brokenType uint64 146 | 147 | const ( 148 | // no errors 149 | brokenNone brokenType = 0 150 | // packfile has read errors 151 | brokenPackfile brokenType = 1 << iota 152 | // there's no index for one packfile 153 | brokenIndex 154 | 155 | packFileGlob = "objects/pack/pack-*.pack" 156 | packBrokenName = "pack-ffffffffffffffffffffffffffffffffffffffff.pack" 157 | ) 158 | 159 | func NewBrokenFS(b brokenType, fs billy.Filesystem) billy.Filesystem { 160 | return &BrokenFS{ 161 | Filesystem: fs, 162 | brokenType: b, 163 | } 164 | } 165 | 166 | type BrokenFS struct { 167 | billy.Filesystem 168 | brokenType brokenType 169 | } 170 | 171 | func (fs *BrokenFS) Open(filename string) (billy.File, error) { 172 | return fs.OpenFile(filename, os.O_RDONLY, 0) 173 | } 174 | 175 | func (fs *BrokenFS) OpenFile( 176 | name string, 177 | flag int, 178 | perm os.FileMode, 179 | ) (billy.File, error) { 180 | file, err := fs.Filesystem.OpenFile(name, flag, perm) 181 | if err != nil { 182 | return nil, err 183 | } 184 | 185 | if fs.brokenType&brokenPackfile == 0 { 186 | return file, err 187 | } 188 | 189 | match, err := filepath.Match(packFileGlob, name) 190 | if err != nil { 191 | return nil, err 192 | } 193 | 194 | if !match { 195 | return file, nil 196 | } 197 | 198 | return &BrokenFile{ 199 | File: file, 200 | }, nil 201 | } 202 | 203 | func (fs *BrokenFS) ReadDir(path string) ([]os.FileInfo, error) { 204 | files, err := fs.Filesystem.ReadDir(path) 205 | if err != nil { 206 | return nil, err 207 | } 208 | 209 | if fs.brokenType&brokenIndex != 0 { 210 | dummyPack := &brokenFileInfo{packBrokenName} 211 | files = append(files, dummyPack) 212 | } 213 | 214 | return files, err 215 | } 216 | 217 | func (fs *BrokenFS) Stat(path string) (os.FileInfo, error) { 218 | stat, err := fs.Filesystem.Stat(path) 219 | return stat, err 220 | } 221 | 222 | type BrokenFile struct { 223 | billy.File 224 | count int 225 | } 226 | 227 | func (fs *BrokenFile) Read(p []byte) (int, error) { 228 | _, err := fs.Seek(0, os.SEEK_CUR) 229 | if err != nil { 230 | return 0, err 231 | } 232 | 233 | fs.count++ 234 | 235 | if fs.count == 10 { 236 | return 0, fmt.Errorf("could not read from broken file") 237 | } 238 | 239 | return fs.File.Read(p) 240 | } 241 | 242 | type brokenFileInfo struct { 243 | name string 244 | } 245 | 246 | func (b *brokenFileInfo) Name() string { 247 | return b.name 248 | } 249 | 250 | func (b *brokenFileInfo) Size() int64 { 251 | return 1024 * 1024 252 | } 253 | 254 | func (b *brokenFileInfo) Mode() os.FileMode { 255 | return 0600 256 | } 257 | 258 | func (b *brokenFileInfo) ModTime() time.Time { 259 | return time.Now() 260 | } 261 | 262 | func (b *brokenFileInfo) IsDir() bool { 263 | return false 264 | } 265 | 266 | func (b *brokenFileInfo) Sys() interface{} { 267 | return nil 268 | } 269 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/src-d/gitbase 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/bblfsh/go-client/v4 v4.1.0 7 | github.com/bblfsh/sdk/v3 v3.2.2 8 | github.com/go-kit/kit v0.8.0 9 | github.com/go-sql-driver/mysql v1.4.1 10 | github.com/gorilla/handlers v1.4.0 // indirect 11 | github.com/hhatto/gocloc v0.3.0 12 | github.com/jessevdk/go-flags v1.4.0 13 | github.com/miekg/dns v1.1.1 // indirect 14 | github.com/opentracing/opentracing-go v1.1.0 15 | github.com/prometheus/client_golang v1.0.0 16 | github.com/sirupsen/logrus v1.4.2 17 | github.com/src-d/enry/v2 v2.0.0 18 | github.com/src-d/go-borges v0.1.4-0.20191017133700-c26ddd90fcbd 19 | github.com/src-d/go-git v4.7.0+incompatible 20 | github.com/src-d/go-git-fixtures v3.5.1-0.20190605154830-57f3972b0248+incompatible 21 | github.com/src-d/go-mysql-server v0.6.1-0.20191028091010-5b6820662f02 22 | github.com/stretchr/testify v1.3.0 23 | github.com/uber-go/atomic v1.4.0 // indirect 24 | github.com/uber/jaeger-client-go v2.16.0+incompatible 25 | github.com/uber/jaeger-lib v2.0.0+incompatible // indirect 26 | go.uber.org/atomic v1.4.0 // indirect 27 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect 28 | golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582 // indirect 29 | golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect 30 | google.golang.org/grpc v1.20.1 31 | gopkg.in/src-d/go-billy-siva.v4 v4.6.0 32 | gopkg.in/src-d/go-billy.v4 v4.3.2 33 | gopkg.in/src-d/go-errors.v1 v1.0.0 34 | gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 35 | gopkg.in/src-d/go-git.v4 v4.13.1 36 | gopkg.in/yaml.v2 v2.2.2 37 | vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible 38 | ) 39 | -------------------------------------------------------------------------------- /index.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "bytes" 5 | "compress/zlib" 6 | "crypto/sha1" 7 | "encoding/binary" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "sync" 12 | 13 | errors "gopkg.in/src-d/go-errors.v1" 14 | "github.com/src-d/go-mysql-server/sql" 15 | ) 16 | 17 | var ( 18 | // ErrColumnNotFound is returned when a given column is not found in the table's schema. 19 | ErrColumnNotFound = errors.NewKind("column %s not found for table %s") 20 | // ErrInvalidObjectType is returned when the received object is not of the correct type. 21 | ErrInvalidObjectType = errors.NewKind("got object of type %T, expecting %s") 22 | ) 23 | 24 | // Indexable represents an indexable gitbase table. 25 | type Indexable interface { 26 | sql.IndexableTable 27 | gitBase 28 | } 29 | 30 | type zlibEncoder struct { 31 | w *zlib.Writer 32 | mut sync.Mutex 33 | } 34 | 35 | func (e *zlibEncoder) encode(data []byte) ([]byte, error) { 36 | e.mut.Lock() 37 | defer e.mut.Unlock() 38 | 39 | var buf bytes.Buffer 40 | e.w.Reset(&buf) 41 | 42 | if _, err := e.w.Write(data); err != nil { 43 | return nil, err 44 | } 45 | 46 | if err := e.w.Close(); err != nil { 47 | return nil, err 48 | } 49 | 50 | return buf.Bytes(), nil 51 | } 52 | 53 | var encoder = func() *zlibEncoder { 54 | return &zlibEncoder{w: zlib.NewWriter(bytes.NewBuffer(nil))} 55 | }() 56 | 57 | func encodeIndexKey(k indexKey) ([]byte, error) { 58 | bs, err := k.encode() 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | return encoder.encode(bs) 64 | } 65 | 66 | func decodeIndexKey(data []byte, k indexKey) error { 67 | gz, err := zlib.NewReader(bytes.NewReader(data)) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | bs, err := ioutil.ReadAll(gz) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | return k.decode(bs) 78 | } 79 | 80 | func rowIndexValues(row sql.Row, columns []string, schema sql.Schema) ([]interface{}, error) { 81 | var values = make([]interface{}, len(columns)) 82 | for i, col := range columns { 83 | var found bool 84 | for j, c := range schema { 85 | if c.Name == col { 86 | values[i] = row[j] 87 | found = true 88 | break 89 | } 90 | } 91 | 92 | if !found { 93 | return nil, ErrColumnNotFound.New(col, schema[0].Source) 94 | } 95 | } 96 | return values, nil 97 | } 98 | 99 | type rowKeyMapper interface { 100 | toRow([]byte) (sql.Row, error) 101 | fromRow(sql.Row) ([]byte, error) 102 | } 103 | 104 | var ( 105 | errRowKeyMapperRowLength = errors.NewKind("row should have %d columns, has: %d") 106 | errRowKeyMapperColType = errors.NewKind("row column %d should have type %T, has: %T") 107 | ) 108 | 109 | type rowIndexIter struct { 110 | mapper rowKeyMapper 111 | index sql.IndexValueIter 112 | } 113 | 114 | func (i *rowIndexIter) Next() (sql.Row, error) { 115 | var err error 116 | var data []byte 117 | defer closeIndexOnError(&err, i.index) 118 | 119 | data, err = i.index.Next() 120 | if err != nil { 121 | return nil, err 122 | } 123 | 124 | row, err := i.mapper.toRow(data) 125 | if err != nil { 126 | return nil, err 127 | } 128 | 129 | return row, nil 130 | } 131 | 132 | func (i *rowIndexIter) Close() error { return i.index.Close() } 133 | 134 | type indexKey interface { 135 | encode() ([]byte, error) 136 | decode([]byte) error 137 | } 138 | 139 | type packOffsetIndexKey struct { 140 | Repository string 141 | Packfile string 142 | Offset int64 143 | Hash string 144 | } 145 | 146 | func (k *packOffsetIndexKey) decode(data []byte) error { 147 | buf := bytes.NewBuffer(data) 148 | var err error 149 | 150 | if k.Repository, err = readString(buf); err != nil { 151 | return err 152 | } 153 | 154 | if k.Packfile, err = readHash(buf); err != nil { 155 | return err 156 | } 157 | 158 | ok, err := readBool(buf) 159 | if err != nil { 160 | return err 161 | } 162 | 163 | if ok { 164 | if k.Offset, err = readInt64(buf); err != nil { 165 | return err 166 | } 167 | k.Hash = "" 168 | } else { 169 | k.Offset = -1 170 | if k.Hash, err = readHash(buf); err != nil { 171 | return err 172 | } 173 | } 174 | 175 | return nil 176 | } 177 | 178 | func (k *packOffsetIndexKey) encode() ([]byte, error) { 179 | var buf bytes.Buffer 180 | writeString(&buf, k.Repository) 181 | if err := writeHash(&buf, k.Packfile); err != nil { 182 | return nil, err 183 | } 184 | writeBool(&buf, k.Offset >= 0) 185 | if k.Offset >= 0 { 186 | writeInt64(&buf, k.Offset) 187 | } else { 188 | if err := writeHash(&buf, k.Hash); err != nil { 189 | return nil, err 190 | } 191 | } 192 | return buf.Bytes(), nil 193 | } 194 | 195 | func readInt64(buf *bytes.Buffer) (int64, error) { 196 | var bs = make([]byte, 8) 197 | _, err := io.ReadFull(buf, bs) 198 | if err != nil { 199 | return 0, fmt.Errorf("can't read int64: %s", err) 200 | } 201 | 202 | ux := binary.LittleEndian.Uint64(bs) 203 | x := int64(ux >> 1) 204 | if ux&1 != 0 { 205 | x = ^x 206 | } 207 | 208 | return x, nil 209 | } 210 | 211 | func readString(buf *bytes.Buffer) (string, error) { 212 | size, err := readInt64(buf) 213 | if err != nil { 214 | return "", fmt.Errorf("can't read string size: %s", err) 215 | } 216 | 217 | var b = make([]byte, int(size)) 218 | if _, err = io.ReadFull(buf, b); err != nil { 219 | return "", fmt.Errorf("can't read string of size %d: %s", size, err) 220 | } 221 | 222 | return string(b), nil 223 | } 224 | 225 | func readBool(buf *bytes.Buffer) (bool, error) { 226 | b, err := buf.ReadByte() 227 | if err != nil { 228 | return false, fmt.Errorf("can't read bool: %s", err) 229 | } 230 | 231 | return b == 1, nil 232 | } 233 | 234 | func writeInt64(buf *bytes.Buffer, n int64) { 235 | ux := uint64(n) << 1 236 | if n < 0 { 237 | ux = ^ux 238 | } 239 | 240 | var bs = make([]byte, 8) 241 | binary.LittleEndian.PutUint64(bs, ux) 242 | buf.Write(bs) 243 | } 244 | 245 | func writeString(buf *bytes.Buffer, s string) { 246 | bs := []byte(s) 247 | writeInt64(buf, int64(len(bs))) 248 | buf.Write(bs) 249 | } 250 | 251 | var ( 252 | hashSize = 2 * sha1.Size 253 | errInvalidHashSize = errors.NewKind("invalid hash size: %d, expecting 40 bytes") 254 | ) 255 | 256 | func writeHash(buf *bytes.Buffer, s string) error { 257 | bs := []byte(s) 258 | if len(bs) != hashSize { 259 | return errInvalidHashSize.New(len(bs)) 260 | } 261 | buf.Write(bs) 262 | return nil 263 | } 264 | 265 | func readHash(buf *bytes.Buffer) (string, error) { 266 | bs := make([]byte, hashSize) 267 | n, err := io.ReadFull(buf, bs) 268 | if err != nil { 269 | return "", fmt.Errorf("can't read hash, only read %d: %s", n, err) 270 | } 271 | return string(bs), nil 272 | } 273 | 274 | func writeBool(buf *bytes.Buffer, b bool) { 275 | if b { 276 | buf.WriteByte(1) 277 | } else { 278 | buf.WriteByte(0) 279 | } 280 | } 281 | 282 | func closeIndexOnError(err *error, index sql.IndexValueIter) { 283 | if *err != nil { 284 | _ = index.Close() 285 | } 286 | } 287 | -------------------------------------------------------------------------------- /index_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "testing" 7 | 8 | "github.com/src-d/go-mysql-server/sql" 9 | "github.com/stretchr/testify/require" 10 | "gopkg.in/src-d/go-git.v4/plumbing" 11 | ) 12 | 13 | func assertEncodeKey(t *testing.T, key indexKey) []byte { 14 | data, err := encodeIndexKey(key) 15 | require.NoError(t, err) 16 | return data 17 | } 18 | 19 | type partitionIndexLookup map[string]sql.IndexValueIter 20 | 21 | func (l partitionIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { 22 | return l[string(p.Key())], nil 23 | } 24 | 25 | func (partitionIndexLookup) Indexes() []string { return nil } 26 | 27 | type indexValueIter struct { 28 | values [][]byte 29 | pos int 30 | } 31 | 32 | func newIndexValueIter(values ...[]byte) sql.IndexValueIter { 33 | return &indexValueIter{values, 0} 34 | } 35 | 36 | func (i *indexValueIter) Next() ([]byte, error) { 37 | if i.pos >= len(i.values) { 38 | return nil, io.EOF 39 | } 40 | 41 | v := i.values[i.pos] 42 | i.pos++ 43 | return v, nil 44 | } 45 | 46 | func (i *indexValueIter) Close() error { return nil } 47 | 48 | type keyValue struct { 49 | key []byte 50 | values []interface{} 51 | } 52 | 53 | func assertIndexKeyValueIter(t *testing.T, iter sql.PartitionIndexKeyValueIter, expected []keyValue) { 54 | t.Helper() 55 | require := require.New(t) 56 | 57 | var result []keyValue 58 | for { 59 | _, kviter, err := iter.Next() 60 | if err == io.EOF { 61 | break 62 | } 63 | require.NoError(err) 64 | 65 | for { 66 | values, key, err := kviter.Next() 67 | if err == io.EOF { 68 | break 69 | } 70 | require.NoError(err) 71 | 72 | result = append(result, keyValue{key, values}) 73 | } 74 | 75 | require.NoError(kviter.Close()) 76 | } 77 | 78 | require.NoError(iter.Close()) 79 | require.Equal(len(expected), len(result)) 80 | require.ElementsMatch(expected, result) 81 | } 82 | 83 | func tableIndexLookup( 84 | t *testing.T, 85 | table sql.IndexableTable, 86 | ctx *sql.Context, 87 | ) sql.IndexLookup { 88 | t.Helper() 89 | iter, err := table.IndexKeyValues(ctx, nil) 90 | require.NoError(t, err) 91 | 92 | lookup := make(partitionIndexLookup) 93 | for { 94 | p, kvIter, err := iter.Next() 95 | if err == io.EOF { 96 | break 97 | } 98 | require.NoError(t, err) 99 | 100 | var values [][]byte 101 | for { 102 | _, val, err := kvIter.Next() 103 | if err == io.EOF { 104 | break 105 | } 106 | require.NoError(t, err) 107 | values = append(values, val) 108 | } 109 | require.NoError(t, kvIter.Close()) 110 | lookup[string(p.Key())] = newIndexValueIter(values...) 111 | } 112 | 113 | require.NoError(t, iter.Close()) 114 | 115 | return lookup 116 | } 117 | 118 | func testTableIndex( 119 | t *testing.T, 120 | table Table, 121 | filters []sql.Expression, 122 | ) { 123 | t.Helper() 124 | require := require.New(t) 125 | ctx, _, cleanup := setup(t) 126 | defer cleanup() 127 | 128 | expected, err := tableToRows(ctx, table) 129 | require.NoError(err) 130 | 131 | indexable := table.(sql.IndexableTable) 132 | lookup := tableIndexLookup(t, indexable, ctx) 133 | tbl := table.(sql.IndexableTable).WithIndexLookup(lookup) 134 | 135 | rows, err := tableToRows(ctx, tbl) 136 | require.NoError(err) 137 | 138 | require.ElementsMatch(expected, rows) 139 | 140 | expected, err = tableToRows(ctx, table.WithFilters(filters)) 141 | require.NoError(err) 142 | 143 | lookup = tableIndexLookup(t, indexable, ctx) 144 | tbl = table.WithFilters(filters).(sql.IndexableTable).WithIndexLookup(lookup) 145 | 146 | rows, err = tableToRows(ctx, tbl) 147 | require.NoError(err) 148 | 149 | require.ElementsMatch(expected, rows) 150 | } 151 | 152 | func TestEncodeRoundtrip(t *testing.T) { 153 | require := require.New(t) 154 | 155 | k := &packOffsetIndexKey{ 156 | Repository: "/foo/bar/baz/repo.git", 157 | Packfile: plumbing.ZeroHash.String(), 158 | Offset: 12345, 159 | Hash: "", 160 | } 161 | 162 | bs, err := encodeIndexKey(k) 163 | require.NoError(err) 164 | 165 | bsraw, err := k.encode() 166 | require.NoError(err) 167 | 168 | // check encodeIndexKey also compresses the encoded value 169 | require.True(len(bs) < len(bsraw)) 170 | 171 | var k2 packOffsetIndexKey 172 | require.NoError(decodeIndexKey(bs, &k2)) 173 | 174 | require.Equal(k, &k2) 175 | } 176 | 177 | func TestWriteReadInt64(t *testing.T) { 178 | require := require.New(t) 179 | 180 | var buf bytes.Buffer 181 | writeInt64(&buf, -7) 182 | 183 | n, err := readInt64(&buf) 184 | require.NoError(err) 185 | require.Equal(int64(-7), n) 186 | 187 | _, err = buf.ReadByte() 188 | require.Equal(err, io.EOF) 189 | 190 | buf.Reset() 191 | writeInt64(&buf, 7) 192 | 193 | n, err = readInt64(&buf) 194 | require.NoError(err) 195 | require.Equal(int64(7), n) 196 | 197 | _, err = buf.ReadByte() 198 | require.Equal(err, io.EOF) 199 | } 200 | 201 | func TestWriteReadBool(t *testing.T) { 202 | require := require.New(t) 203 | 204 | var buf bytes.Buffer 205 | writeBool(&buf, true) 206 | 207 | b, err := readBool(&buf) 208 | require.NoError(err) 209 | require.True(b) 210 | 211 | _, err = buf.ReadByte() 212 | require.Equal(err, io.EOF) 213 | } 214 | 215 | func TestWriteReadString(t *testing.T) { 216 | require := require.New(t) 217 | 218 | var buf bytes.Buffer 219 | writeString(&buf, "foo bar") 220 | 221 | s, err := readString(&buf) 222 | require.NoError(err) 223 | require.Equal("foo bar", s) 224 | 225 | _, err = buf.ReadByte() 226 | require.Equal(err, io.EOF) 227 | } 228 | 229 | func TestWriteReadHash(t *testing.T) { 230 | require := require.New(t) 231 | 232 | var buf bytes.Buffer 233 | 234 | require.Error(writeHash(&buf, "")) 235 | require.NoError(writeHash(&buf, plumbing.ZeroHash.String())) 236 | 237 | h, err := readHash(&buf) 238 | require.NoError(err) 239 | require.Equal(plumbing.ZeroHash.String(), h) 240 | 241 | _, err = buf.ReadByte() 242 | require.Equal(err, io.EOF) 243 | } 244 | 245 | func TestEncodePackOffsetIndexKey(t *testing.T) { 246 | require := require.New(t) 247 | 248 | k := packOffsetIndexKey{ 249 | Repository: "repo1", 250 | Packfile: plumbing.ZeroHash.String(), 251 | Offset: 1234, 252 | Hash: "", 253 | } 254 | 255 | data, err := k.encode() 256 | require.NoError(err) 257 | 258 | var k2 packOffsetIndexKey 259 | require.NoError(k2.decode(data)) 260 | 261 | require.Equal(k, k2) 262 | 263 | k = packOffsetIndexKey{ 264 | Repository: "repo1", 265 | Packfile: plumbing.ZeroHash.String(), 266 | Offset: -1, 267 | Hash: plumbing.ZeroHash.String(), 268 | } 269 | 270 | data, err = k.encode() 271 | require.NoError(err) 272 | 273 | var k3 packOffsetIndexKey 274 | require.NoError(k3.decode(data)) 275 | 276 | require.Equal(k, k3) 277 | } 278 | -------------------------------------------------------------------------------- /init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -n "$SIVA" ]; then 4 | SIVA_ARGS="--format siva --bucket 2" 5 | fi 6 | 7 | cat <> "$HOME/.my.cnf" 8 | [client] 9 | user=${GITBASE_USER} 10 | password=${GITBASE_PASSWORD} 11 | EOT 12 | 13 | export GITBASE_LOG_LEVEL=${GITBASE_LOG_LEVEL:-debug} 14 | 15 | /tini -s -- /bin/gitbase server \ 16 | --host=0.0.0.0 \ 17 | --port=3306 \ 18 | --user="$GITBASE_USER" \ 19 | --password="$GITBASE_PASSWORD" \ 20 | --directories="$GITBASE_REPOS" \ 21 | $SIVA_ARGS 22 | -------------------------------------------------------------------------------- /internal/commitstats/commit.go: -------------------------------------------------------------------------------- 1 | package commitstats 2 | 3 | import ( 4 | "fmt" 5 | 6 | "gopkg.in/src-d/go-git.v4" 7 | "gopkg.in/src-d/go-git.v4/plumbing/object" 8 | ) 9 | 10 | // CommitStats represents the stats for a commit. 11 | type CommitStats struct { 12 | // Files add/modified/removed by this commit. 13 | Files int 14 | // Code stats of the code lines. 15 | Code KindStats 16 | // Comment stats of the comment lines. 17 | Comment KindStats 18 | // Blank stats of the blank lines. 19 | Blank KindStats 20 | // Other stats of files that are not from a recognized or format language. 21 | Other KindStats 22 | // Total the sum of the previous stats. 23 | Total KindStats 24 | } 25 | 26 | func (s *CommitStats) String() string { 27 | return fmt.Sprintf("Code (+%d/-%d)\nComment (+%d/-%d)\nBlank (+%d/-%d)\nOther (+%d/-%d)\nTotal (+%d/-%d)\nFiles (%d)\n", 28 | s.Code.Additions, s.Code.Deletions, 29 | s.Comment.Additions, s.Comment.Deletions, 30 | s.Blank.Additions, s.Blank.Deletions, 31 | s.Other.Additions, s.Other.Deletions, 32 | s.Total.Additions, s.Total.Deletions, 33 | s.Files, 34 | ) 35 | } 36 | 37 | // Calculate calculates the CommitStats for from commit to another. 38 | // if from is nil the first parent is used, if the commit is orphan the stats 39 | // are compared against a empty commit. 40 | func Calculate(r *git.Repository, from, to *object.Commit) (*CommitStats, error) { 41 | fs, err := CalculateByFile(r, from, to) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | return commitStatsFromCommitFileStats(fs), nil 47 | } 48 | 49 | func commitStatsFromCommitFileStats(fs []CommitFileStats) *CommitStats { 50 | var s CommitStats 51 | for _, f := range fs { 52 | s.Blank.Add(f.Blank) 53 | s.Comment.Add(f.Comment) 54 | s.Code.Add(f.Code) 55 | s.Other.Add(f.Other) 56 | s.Total.Add(f.Total) 57 | s.Files++ 58 | } 59 | return &s 60 | } 61 | -------------------------------------------------------------------------------- /internal/commitstats/commit_test.go: -------------------------------------------------------------------------------- 1 | package commitstats 2 | 3 | import ( 4 | "testing" 5 | 6 | fixtures "github.com/src-d/go-git-fixtures" 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | "gopkg.in/src-d/go-git.v4" 10 | "gopkg.in/src-d/go-git.v4/plumbing" 11 | "gopkg.in/src-d/go-git.v4/plumbing/cache" 12 | "gopkg.in/src-d/go-git.v4/plumbing/object" 13 | "gopkg.in/src-d/go-git.v4/storage/filesystem" 14 | ) 15 | 16 | func TestCalculate(t *testing.T) { 17 | defer func() { 18 | require.NoError(t, fixtures.Clean()) 19 | }() 20 | 21 | tests := map[string]struct { 22 | fixture *fixtures.Fixture 23 | from plumbing.Hash 24 | to plumbing.Hash 25 | expected *CommitStats 26 | }{ 27 | "basic": { 28 | fixture: fixtures.ByURL("https://github.com/src-d/go-git.git").One(), 29 | to: plumbing.NewHash("d2d68d3413353bd4bf20891ac1daa82cd6e00fb9"), 30 | expected: &CommitStats{ 31 | Files: 23, 32 | Code: KindStats{Additions: 414, Deletions: 264}, 33 | Blank: KindStats{Additions: 42, Deletions: 65}, 34 | Comment: KindStats{Additions: 2, Deletions: 337}, 35 | Total: KindStats{Additions: 458, Deletions: 666}, 36 | }, 37 | }, 38 | "orphan": { 39 | fixture: fixtures.Basic().One(), 40 | to: plumbing.NewHash("b029517f6300c2da0f4b651b8642506cd6aaf45d"), 41 | expected: &CommitStats{ 42 | Files: 1, 43 | Other: KindStats{Additions: 22, Deletions: 0}, 44 | Total: KindStats{Additions: 22, Deletions: 0}, 45 | }, 46 | }, 47 | "other": { 48 | fixture: fixtures.Basic().One(), 49 | to: plumbing.NewHash("b8e471f58bcbca63b07bda20e428190409c2db47"), 50 | expected: &CommitStats{ 51 | Files: 1, 52 | Other: KindStats{Additions: 1, Deletions: 0}, 53 | Total: KindStats{Additions: 1, Deletions: 0}, 54 | }, 55 | }, 56 | "binary": { 57 | fixture: fixtures.Basic().One(), 58 | to: plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9"), 59 | expected: &CommitStats{ 60 | Files: 1, 61 | }, 62 | }, 63 | "vendor": { 64 | fixture: fixtures.Basic().One(), 65 | to: plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), 66 | expected: &CommitStats{ 67 | Files: 0, 68 | }, 69 | }, 70 | "with_from": { 71 | fixture: fixtures.Basic().One(), 72 | to: plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), 73 | from: plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), 74 | expected: &CommitStats{}, 75 | }, 76 | } 77 | 78 | for name, test := range tests { 79 | t.Run(name, func(t *testing.T) { 80 | require := require.New(t) 81 | 82 | r, err := git.Open(filesystem.NewStorage(test.fixture.DotGit(), cache.NewObjectLRUDefault()), nil) 83 | require.NoError(err) 84 | 85 | to, err := r.CommitObject(test.to) 86 | require.NoError(err) 87 | 88 | var from *object.Commit 89 | if !test.from.IsZero() { 90 | from, err = r.CommitObject(test.from) 91 | require.NoError(err) 92 | } 93 | 94 | stats, err := Calculate(r, from, to) 95 | require.NoError(err) 96 | 97 | assert.Equal(t, test.expected, stats) 98 | }) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /internal/commitstats/common.go: -------------------------------------------------------------------------------- 1 | package commitstats 2 | 3 | // LineKind defines the kind of a line in a file. 4 | type LineKind int 5 | 6 | const ( 7 | // Code represents a line of code. 8 | Code LineKind = iota + 1 9 | // Comment represents a line of comment. 10 | Comment 11 | // Blank represents an empty line. 12 | Blank 13 | // Other represents a line from any other kind. 14 | Other 15 | ) 16 | 17 | // KindStats represents the stats for a kind of lines in a file. 18 | type KindStats struct { 19 | // Additions number of lines added. 20 | Additions int 21 | // Deletions number of lines deleted. 22 | Deletions int 23 | } 24 | 25 | // Add adds the given stats to this stats. 26 | func (k *KindStats) Add(add KindStats) { 27 | k.Additions += add.Additions 28 | k.Deletions += add.Deletions 29 | } 30 | -------------------------------------------------------------------------------- /internal/function/blame.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/src-d/gitbase" 8 | "github.com/src-d/go-mysql-server/sql" 9 | "gopkg.in/src-d/go-git.v4" 10 | 11 | "gopkg.in/src-d/go-git.v4/plumbing" 12 | "gopkg.in/src-d/go-git.v4/plumbing/object" 13 | ) 14 | 15 | type BlameGenerator struct { 16 | ctx *sql.Context 17 | commit *object.Commit 18 | file string 19 | curLine int 20 | lines []*git.Line 21 | } 22 | 23 | func NewBlameGenerator(ctx *sql.Context, c *object.Commit, f string) (*BlameGenerator, error) { 24 | result, err := git.Blame(c, f) 25 | if err != nil { 26 | return nil, err 27 | } 28 | return &BlameGenerator{ 29 | ctx: ctx, 30 | commit: c, 31 | file: f, 32 | curLine: 0, 33 | lines: result.Lines, 34 | }, nil 35 | } 36 | 37 | func (g *BlameGenerator) Next() (interface{}, error) { 38 | select { 39 | case <-g.ctx.Done(): 40 | return nil, io.EOF 41 | default: 42 | } 43 | 44 | if len(g.lines) == 0 || g.curLine >= len(g.lines) { 45 | return nil, io.EOF 46 | } 47 | 48 | l := g.lines[g.curLine] 49 | b := BlameLine{ 50 | LineNum: g.curLine, 51 | Author: l.Author, 52 | Text: l.Text, 53 | } 54 | g.curLine++ 55 | return b, nil 56 | } 57 | 58 | func (g *BlameGenerator) Close() error { 59 | return nil 60 | } 61 | 62 | var _ sql.Generator = (*BlameGenerator)(nil) 63 | 64 | type ( 65 | // Blame implements git-blame function as UDF 66 | Blame struct { 67 | repo sql.Expression 68 | commit sql.Expression 69 | file sql.Expression 70 | } 71 | 72 | // BlameLine represents each line of git blame's output 73 | BlameLine struct { 74 | LineNum int `json:"linenum"` 75 | Author string `json:"author"` 76 | Text string `json:"text"` 77 | } 78 | ) 79 | 80 | // NewBlame constructor 81 | func NewBlame(repo, commit, file sql.Expression) sql.Expression { 82 | return &Blame{repo, commit, file} 83 | } 84 | 85 | func (b *Blame) String() string { 86 | return fmt.Sprintf("blame(%s, %s)", b.repo, b.commit) 87 | } 88 | 89 | // Type implements the sql.Expression interface 90 | func (*Blame) Type() sql.Type { 91 | return sql.Array(sql.JSON) 92 | } 93 | 94 | func (b *Blame) WithChildren(children ...sql.Expression) (sql.Expression, error) { 95 | if len(children) != 3 { 96 | return nil, sql.ErrInvalidChildrenNumber.New(b, len(children), 2) 97 | } 98 | 99 | return NewBlame(children[0], children[1], children[2]), nil 100 | } 101 | 102 | // Children implements the Expression interface. 103 | func (b *Blame) Children() []sql.Expression { 104 | return []sql.Expression{b.repo, b.commit, b.file} 105 | } 106 | 107 | // IsNullable implements the Expression interface. 108 | func (b *Blame) IsNullable() bool { 109 | return b.repo.IsNullable() || (b.commit.IsNullable()) || (b.file.IsNullable()) 110 | } 111 | 112 | // Resolved implements the Expression interface. 113 | func (b *Blame) Resolved() bool { 114 | return b.repo.Resolved() && b.commit.Resolved() && b.file.Resolved() 115 | } 116 | 117 | // Eval implements the sql.Expression interface. 118 | func (b *Blame) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { 119 | span, ctx := ctx.Span("gitbase.Blame") 120 | defer span.Finish() 121 | 122 | repo, err := b.resolveRepo(ctx, row) 123 | if err != nil { 124 | ctx.Warn(0, err.Error()) 125 | return nil, nil 126 | } 127 | 128 | commit, err := b.resolveCommit(ctx, repo, row) 129 | if err != nil { 130 | ctx.Warn(0, err.Error()) 131 | return nil, nil 132 | } 133 | 134 | file, err := exprToString(ctx, b.file, row) 135 | if err != nil { 136 | ctx.Warn(0, err.Error()) 137 | return nil, nil 138 | } 139 | 140 | bg, err := NewBlameGenerator(ctx, commit, file) 141 | if err != nil { 142 | ctx.Warn(0, err.Error()) 143 | return nil, nil 144 | } 145 | 146 | return bg, nil 147 | } 148 | 149 | func (b *Blame) resolveCommit(ctx *sql.Context, repo *gitbase.Repository, row sql.Row) (*object.Commit, error) { 150 | str, err := exprToString(ctx, b.commit, row) 151 | if err != nil { 152 | return nil, err 153 | } 154 | 155 | commitHash, err := repo.ResolveRevision(plumbing.Revision(str)) 156 | if err != nil { 157 | h := plumbing.NewHash(str) 158 | commitHash = &h 159 | } 160 | to, err := repo.CommitObject(*commitHash) 161 | if err != nil { 162 | return nil, err 163 | } 164 | 165 | return to, nil 166 | } 167 | 168 | func (b *Blame) resolveRepo(ctx *sql.Context, r sql.Row) (*gitbase.Repository, error) { 169 | repoID, err := exprToString(ctx, b.repo, r) 170 | if err != nil { 171 | return nil, err 172 | } 173 | s, ok := ctx.Session.(*gitbase.Session) 174 | if !ok { 175 | return nil, gitbase.ErrInvalidGitbaseSession.New(ctx.Session) 176 | } 177 | return s.Pool.GetRepo(repoID) 178 | } 179 | -------------------------------------------------------------------------------- /internal/function/blame_test.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/src-d/gitbase" 8 | "github.com/src-d/go-mysql-server/sql" 9 | "github.com/src-d/go-mysql-server/sql/expression" 10 | "github.com/stretchr/testify/require" 11 | fixtures "gopkg.in/src-d/go-git-fixtures.v3" 12 | ) 13 | 14 | func TestBlameEval(t *testing.T) { 15 | require.NoError(t, fixtures.Init()) 16 | 17 | defer func() { 18 | require.NoError(t, fixtures.Clean()) 19 | }() 20 | 21 | pool, cleanup := setupPool(t) 22 | defer cleanup() 23 | 24 | session := gitbase.NewSession(pool) 25 | ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) 26 | 27 | testCases := []struct { 28 | name string 29 | repo sql.Expression 30 | commit sql.Expression 31 | file sql.Expression 32 | row sql.Row 33 | expected BlameLine 34 | expectedNil bool 35 | testedLine int 36 | lineCount int 37 | }{ 38 | { 39 | name: "init commit", 40 | repo: expression.NewGetField(0, sql.Text, "repository_id", false), 41 | commit: expression.NewGetField(1, sql.Text, "commit_hash", false), 42 | file: expression.NewGetField(2, sql.Text, "file", false), 43 | row: sql.NewRow("worktree", "b029517f6300c2da0f4b651b8642506cd6aaf45d", ".gitignore"), 44 | testedLine: 0, 45 | lineCount: 12, 46 | expected: BlameLine{ 47 | 0, 48 | "mcuadros@gmail.com", 49 | "*.class", 50 | }, 51 | expectedNil: false, 52 | }, 53 | { 54 | name: "changelog", 55 | repo: expression.NewGetField(0, sql.Text, "repository_id", false), 56 | commit: expression.NewGetField(1, sql.Text, "commit_hash", false), 57 | file: expression.NewGetField(2, sql.Text, "file", false), 58 | row: sql.NewRow("worktree", "b8e471f58bcbca63b07bda20e428190409c2db47", "CHANGELOG"), 59 | testedLine: 0, 60 | lineCount: 1, 61 | expected: BlameLine{ 62 | 0, 63 | "daniel@lordran.local", 64 | "Initial changelog", 65 | }, 66 | expectedNil: false, 67 | }, 68 | { 69 | name: "no repo", 70 | repo: expression.NewGetField(0, sql.Text, "repository_id", false), 71 | commit: expression.NewGetField(1, sql.Text, "commit_hash", false), 72 | file: expression.NewGetField(2, sql.Text, "file", false), 73 | row: sql.NewRow("foo", "bar", "baz"), 74 | testedLine: 0, 75 | lineCount: 1, 76 | expected: BlameLine{}, 77 | expectedNil: true, 78 | }, 79 | { 80 | name: "no commit", 81 | repo: expression.NewGetField(0, sql.Text, "repository_id", false), 82 | commit: expression.NewGetField(1, sql.Text, "commit_hash", false), 83 | file: expression.NewGetField(2, sql.Text, "file", false), 84 | row: sql.NewRow("worktree", "foo", "bar"), 85 | testedLine: 0, 86 | lineCount: 1, 87 | expected: BlameLine{}, 88 | expectedNil: true, 89 | }, 90 | { 91 | name: "no file", 92 | repo: expression.NewGetField(0, sql.Text, "repository_id", false), 93 | commit: expression.NewGetField(1, sql.Text, "commit_hash", false), 94 | file: expression.NewGetField(2, sql.Text, "file", false), 95 | row: sql.NewRow("worktree", "b8e471f58bcbca63b07bda20e428190409c2db47", "foo"), 96 | testedLine: 0, 97 | lineCount: 1, 98 | expected: BlameLine{}, 99 | expectedNil: true, 100 | }, 101 | } 102 | 103 | for _, tc := range testCases { 104 | t.Run(tc.name, func(t *testing.T) { 105 | blame := NewBlame(tc.repo, tc.commit, tc.file) 106 | blameGen, err := blame.Eval(ctx, tc.row) 107 | require.NoError(t, err) 108 | 109 | if tc.expectedNil { 110 | require.Nil(t, blameGen) 111 | return 112 | } else { 113 | require.NotNil(t, blameGen) 114 | } 115 | 116 | bg := blameGen.(*BlameGenerator) 117 | defer bg.Close() 118 | 119 | lineCount := 0 120 | for i, err := bg.Next(); err == nil; i, err = bg.Next() { 121 | i := i.(BlameLine) 122 | if lineCount != tc.testedLine { 123 | lineCount++ 124 | continue 125 | } 126 | lineCount++ 127 | require.EqualValues(t, tc.expected, i) 128 | } 129 | require.Equal(t, tc.lineCount, lineCount) 130 | }) 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /internal/function/commit_file_stats.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/src-d/gitbase/internal/commitstats" 7 | 8 | "github.com/src-d/go-mysql-server/sql" 9 | "gopkg.in/src-d/go-git.v4" 10 | "gopkg.in/src-d/go-git.v4/plumbing/object" 11 | ) 12 | 13 | // CommitFileStats calculates the diff stats of all files for a given commit. 14 | // Vendored files are ignored in the output of this function. 15 | type CommitFileStats struct { 16 | Repository sql.Expression 17 | From sql.Expression 18 | To sql.Expression 19 | } 20 | 21 | // NewCommitFileStats creates a new COMMIT_FILE_STATS function. 22 | func NewCommitFileStats(args ...sql.Expression) (sql.Expression, error) { 23 | var f CommitFileStats 24 | switch len(args) { 25 | case 2: 26 | f.Repository, f.To = args[0], args[1] 27 | case 3: 28 | f.Repository, f.From, f.To = args[0], args[1], args[2] 29 | default: 30 | return nil, sql.ErrInvalidArgumentNumber.New("COMMIT_FILE_STATS", "2 or 3", len(args)) 31 | } 32 | 33 | return &f, nil 34 | } 35 | 36 | func (f *CommitFileStats) String() string { 37 | if f.From == nil { 38 | return fmt.Sprintf("commit_file_stats(%s, %s)", f.Repository, f.To) 39 | } 40 | 41 | return fmt.Sprintf("commit_file_stats(%s, %s, %s)", f.Repository, f.From, f.To) 42 | } 43 | 44 | // Type implements the Expression interface. 45 | func (CommitFileStats) Type() sql.Type { 46 | return sql.Array(sql.JSON) 47 | } 48 | 49 | // WithChildren implements the Expression interface. 50 | func (f *CommitFileStats) WithChildren(children ...sql.Expression) (sql.Expression, error) { 51 | expected := 2 52 | if f.From != nil { 53 | expected = 3 54 | } 55 | 56 | if len(children) != expected { 57 | return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), expected) 58 | } 59 | 60 | repo := children[0] 61 | var from, to sql.Expression 62 | if f.From != nil { 63 | from = children[1] 64 | to = children[2] 65 | } else { 66 | to = children[1] 67 | } 68 | 69 | return &CommitFileStats{repo, from, to}, nil 70 | } 71 | 72 | // Children implements the Expression interface. 73 | func (f *CommitFileStats) Children() []sql.Expression { 74 | if f.From == nil { 75 | return []sql.Expression{f.Repository, f.To} 76 | } 77 | 78 | return []sql.Expression{f.Repository, f.From, f.To} 79 | } 80 | 81 | // IsNullable implements the Expression interface. 82 | func (*CommitFileStats) IsNullable() bool { 83 | return true 84 | } 85 | 86 | // Resolved implements the Expression interface. 87 | func (f *CommitFileStats) Resolved() bool { 88 | return f.Repository.Resolved() && 89 | f.To.Resolved() && 90 | (f.From == nil || f.From.Resolved()) 91 | } 92 | 93 | // Eval implements the Expression interface. 94 | func (f *CommitFileStats) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { 95 | return evalStatsFunc( 96 | ctx, 97 | "commit_file_stats", 98 | row, 99 | f.Repository, f.From, f.To, 100 | func(r *git.Repository, from, to *object.Commit) (interface{}, error) { 101 | stats, err := commitstats.CalculateByFile(r, from, to) 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | // Since the type is an array, it must be converted to []interface{}. 107 | var result = make([]interface{}, len(stats)) 108 | for i, s := range stats { 109 | result[i] = s 110 | } 111 | return result, nil 112 | }, 113 | ) 114 | } 115 | -------------------------------------------------------------------------------- /internal/function/commit_file_stats_test.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "context" 5 | 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/src-d/gitbase" 10 | "github.com/src-d/gitbase/internal/commitstats" 11 | "github.com/stretchr/testify/require" 12 | 13 | "github.com/src-d/go-mysql-server/sql" 14 | "github.com/src-d/go-mysql-server/sql/expression" 15 | ) 16 | 17 | func TestCommitFileStats(t *testing.T) { 18 | pool, cleanup := setupPool(t) 19 | defer cleanup() 20 | 21 | session := gitbase.NewSession(pool) 22 | ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) 23 | 24 | testCases := []struct { 25 | name string 26 | repo sql.Expression 27 | from sql.Expression 28 | to sql.Expression 29 | row sql.Row 30 | expected interface{} 31 | }{ 32 | { 33 | name: "init commit", 34 | repo: expression.NewGetField(0, sql.Text, "repository_id", false), 35 | from: nil, 36 | to: expression.NewGetField(1, sql.Text, "commit_hash", false), 37 | row: sql.NewRow("worktree", "b029517f6300c2da0f4b651b8642506cd6aaf45d"), 38 | expected: []interface{}{ 39 | commitstats.CommitFileStats{ 40 | Path: "LICENSE", 41 | Language: "Text", 42 | Other: commitstats.KindStats{Additions: 22}, 43 | Total: commitstats.KindStats{Additions: 22}, 44 | }, 45 | }, 46 | }, 47 | { 48 | name: "invalid repository id", 49 | repo: expression.NewGetField(0, sql.Text, "repository_id", false), 50 | from: nil, 51 | to: expression.NewGetField(1, sql.Text, "commit_hash", false), 52 | row: sql.NewRow("foobar", "b029517f6300c2da0f4b651b8642506cd6aaf45d"), 53 | expected: nil, 54 | }, 55 | { 56 | name: "invalid to", 57 | repo: expression.NewGetField(0, sql.Text, "repository_id", false), 58 | from: nil, 59 | to: expression.NewGetField(1, sql.Text, "commit_hash", false), 60 | row: sql.NewRow("worktree", "foobar"), 61 | expected: nil, 62 | }, 63 | { 64 | name: "invalid from", 65 | repo: expression.NewGetField(0, sql.Text, "repository_id", false), 66 | from: expression.NewGetField(2, sql.Text, "commit_hash", false), 67 | to: expression.NewGetField(1, sql.Text, "commit_hash", false), 68 | row: sql.NewRow("worktree", "b029517f6300c2da0f4b651b8642506cd6aaf45d", "foobar"), 69 | expected: nil, 70 | }, 71 | } 72 | 73 | for _, tc := range testCases { 74 | t.Run(tc.name, func(t *testing.T) { 75 | diff, err := NewCommitFileStats(tc.repo, tc.from, tc.to) 76 | require.NoError(t, err) 77 | 78 | result, err := diff.Eval(ctx, tc.row) 79 | require.NoError(t, err) 80 | 81 | require.EqualValues(t, tc.expected, result) 82 | }) 83 | } 84 | } 85 | 86 | func TestWithChildren(t *testing.T) { 87 | repo := expression.NewGetField(0, sql.Text, "repository_id", false) 88 | from := expression.NewGetField(2, sql.Text, "commit_hash", false) 89 | 90 | cfs, err := NewCommitFileStats(repo, from) 91 | require.NoError(t, err) 92 | 93 | newCfs, err := cfs.WithChildren(repo, from) 94 | 95 | require.NoError(t, err) 96 | require.EqualValues(t, cfs.Children(), newCfs.Children()) 97 | require.Equal(t, reflect.TypeOf(cfs), reflect.TypeOf(newCfs)) 98 | } 99 | -------------------------------------------------------------------------------- /internal/function/commit_stats.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/sirupsen/logrus" 7 | "github.com/src-d/gitbase" 8 | "github.com/src-d/gitbase/internal/commitstats" 9 | 10 | "github.com/src-d/go-mysql-server/sql" 11 | "gopkg.in/src-d/go-git.v4" 12 | "gopkg.in/src-d/go-git.v4/plumbing" 13 | "gopkg.in/src-d/go-git.v4/plumbing/object" 14 | ) 15 | 16 | // CommitStats calculates the diff stats for a given commit. Vendored files 17 | // are completely ignored for the output of this function. 18 | type CommitStats struct { 19 | Repository sql.Expression 20 | From sql.Expression 21 | To sql.Expression 22 | } 23 | 24 | // NewCommitStats creates a new COMMIT_STATS function. 25 | func NewCommitStats(args ...sql.Expression) (sql.Expression, error) { 26 | f := &CommitStats{} 27 | switch len(args) { 28 | case 2: 29 | f.Repository, f.To = args[0], args[1] 30 | case 3: 31 | f.Repository, f.From, f.To = args[0], args[1], args[2] 32 | default: 33 | return nil, sql.ErrInvalidArgumentNumber.New("COMMIT_STATS", "2 or 3", len(args)) 34 | } 35 | 36 | return f, nil 37 | } 38 | 39 | func (f *CommitStats) String() string { 40 | if f.From == nil { 41 | return fmt.Sprintf("commit_stats(%s, %s)", f.Repository, f.To) 42 | } 43 | 44 | return fmt.Sprintf("commit_stats(%s, %s, %s)", f.Repository, f.From, f.To) 45 | } 46 | 47 | // Type implements the Expression interface. 48 | func (CommitStats) Type() sql.Type { 49 | return sql.JSON 50 | } 51 | 52 | // WithChildren implements the Expression interface. 53 | func (f *CommitStats) WithChildren(children ...sql.Expression) (sql.Expression, error) { 54 | expected := 2 55 | if f.From != nil { 56 | expected = 3 57 | } 58 | 59 | if len(children) != expected { 60 | return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), expected) 61 | } 62 | 63 | repo := children[0] 64 | var from, to sql.Expression 65 | if f.From != nil { 66 | from = children[1] 67 | to = children[2] 68 | } else { 69 | to = children[1] 70 | } 71 | 72 | return &CommitStats{repo, from, to}, nil 73 | } 74 | 75 | // Children implements the Expression interface. 76 | func (f *CommitStats) Children() []sql.Expression { 77 | if f.From == nil { 78 | return []sql.Expression{f.Repository, f.To} 79 | } 80 | 81 | return []sql.Expression{f.Repository, f.From, f.To} 82 | } 83 | 84 | // IsNullable implements the Expression interface. 85 | func (*CommitStats) IsNullable() bool { 86 | return true 87 | } 88 | 89 | // Resolved implements the Expression interface. 90 | func (f *CommitStats) Resolved() bool { 91 | return f.Repository.Resolved() && 92 | f.To.Resolved() && 93 | (f.From == nil || f.From.Resolved()) 94 | } 95 | 96 | // Eval implements the Expression interface. 97 | func (f *CommitStats) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { 98 | return evalStatsFunc( 99 | ctx, 100 | "commit_stats", 101 | row, 102 | f.Repository, f.From, f.To, 103 | func(r *git.Repository, from, to *object.Commit) (interface{}, error) { 104 | return commitstats.Calculate(r, from, to) 105 | }, 106 | ) 107 | } 108 | 109 | func resolveRepo( 110 | ctx *sql.Context, 111 | r sql.Row, 112 | repo sql.Expression, 113 | ) (*gitbase.Repository, error) { 114 | repoID, err := exprToString(ctx, repo, r) 115 | if err != nil { 116 | return nil, err 117 | } 118 | 119 | s, ok := ctx.Session.(*gitbase.Session) 120 | if !ok { 121 | return nil, gitbase.ErrInvalidGitbaseSession.New(ctx.Session) 122 | } 123 | return s.Pool.GetRepo(repoID) 124 | } 125 | 126 | func resolveCommit( 127 | ctx *sql.Context, 128 | r *gitbase.Repository, 129 | row sql.Row, 130 | e sql.Expression, 131 | ) (*object.Commit, error) { 132 | str, err := exprToString(ctx, e, row) 133 | if err != nil { 134 | return nil, err 135 | } 136 | 137 | if str == "" { 138 | return nil, nil 139 | } 140 | 141 | commitHash, err := r.ResolveRevision(plumbing.Revision(str)) 142 | if err != nil { 143 | h := plumbing.NewHash(str) 144 | commitHash = &h 145 | } 146 | 147 | return r.CommitObject(*commitHash) 148 | } 149 | 150 | func evalStatsFunc( 151 | ctx *sql.Context, 152 | name string, 153 | row sql.Row, 154 | repoExpr, fromExpr, toExpr sql.Expression, 155 | fn func(r *git.Repository, from, to *object.Commit) (interface{}, error), 156 | ) (interface{}, error) { 157 | span, ctx := ctx.Span("gitbase." + name) 158 | defer span.Finish() 159 | 160 | r, err := resolveRepo(ctx, row, repoExpr) 161 | if err != nil { 162 | ctx.Warn(0, name+": unable to resolve repository") 163 | logrus.WithField("err", err).Error(name + ": unable to resolve repository") 164 | return nil, nil 165 | } 166 | 167 | log := logrus.WithField("repository", r) 168 | 169 | to, err := resolveCommit(ctx, r, row, toExpr) 170 | if err != nil { 171 | ctx.Warn(0, name+": unable to resolve 'to' commit of repository: %v", r) 172 | log.WithField("err", err).Error(name + ": unable to resolve 'to' commit") 173 | return nil, nil 174 | } 175 | 176 | from, err := resolveCommit(ctx, r, row, fromExpr) 177 | if err != nil { 178 | ctx.Warn(0, name+": unable to resolve 'from' commit of repository: %v", r) 179 | log.WithField("err", err).Error(name + ": unable to resolve from commit") 180 | return nil, nil 181 | } 182 | 183 | result, err := fn(r.Repository, from, to) 184 | if err != nil { 185 | ctx.Warn(0, name+": unable to calculate for repository: %v, from: %v, to: %v", r, from, to) 186 | log.WithFields(logrus.Fields{ 187 | "err": err, 188 | "from": from, 189 | "to": to, 190 | }).Error(name + ": unable to calculate") 191 | return nil, nil 192 | } 193 | 194 | return result, nil 195 | } 196 | -------------------------------------------------------------------------------- /internal/function/commit_stats_test.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/src-d/gitbase" 10 | "github.com/src-d/gitbase/internal/commitstats" 11 | "github.com/src-d/go-borges/plain" 12 | "github.com/stretchr/testify/require" 13 | 14 | fixtures "github.com/src-d/go-git-fixtures" 15 | "github.com/src-d/go-mysql-server/sql" 16 | "github.com/src-d/go-mysql-server/sql/expression" 17 | "gopkg.in/src-d/go-billy.v4/osfs" 18 | "gopkg.in/src-d/go-git.v4/plumbing/cache" 19 | ) 20 | 21 | func TestCommitStatsEval(t *testing.T) { 22 | pool, cleanup := setupPool(t) 23 | defer cleanup() 24 | 25 | session := gitbase.NewSession(pool) 26 | ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) 27 | 28 | testCases := []struct { 29 | name string 30 | repo sql.Expression 31 | from sql.Expression 32 | to sql.Expression 33 | row sql.Row 34 | expected interface{} 35 | }{ 36 | { 37 | name: "init commit", 38 | repo: expression.NewGetField(0, sql.Text, "repository_id", false), 39 | from: nil, 40 | to: expression.NewGetField(1, sql.Text, "commit_hash", false), 41 | row: sql.NewRow("worktree", "b029517f6300c2da0f4b651b8642506cd6aaf45d"), 42 | expected: &commitstats.CommitStats{ 43 | Files: 1, 44 | Other: commitstats.KindStats{Additions: 22, Deletions: 0}, 45 | Total: commitstats.KindStats{Additions: 22, Deletions: 0}, 46 | }, 47 | }, 48 | { 49 | name: "invalid repository id", 50 | repo: expression.NewGetField(0, sql.Text, "repository_id", false), 51 | from: nil, 52 | to: expression.NewGetField(1, sql.Text, "commit_hash", false), 53 | row: sql.NewRow("foobar", "b029517f6300c2da0f4b651b8642506cd6aaf45d"), 54 | expected: nil, 55 | }, 56 | { 57 | name: "invalid to", 58 | repo: expression.NewGetField(0, sql.Text, "repository_id", false), 59 | from: nil, 60 | to: expression.NewGetField(1, sql.Text, "commit_hash", false), 61 | row: sql.NewRow("worktree", "foobar"), 62 | expected: nil, 63 | }, 64 | { 65 | name: "invalid from", 66 | repo: expression.NewGetField(0, sql.Text, "repository_id", false), 67 | from: expression.NewGetField(2, sql.Text, "commit_hash", false), 68 | to: expression.NewGetField(1, sql.Text, "commit_hash", false), 69 | row: sql.NewRow("worktree", "b029517f6300c2da0f4b651b8642506cd6aaf45d", "foobar"), 70 | expected: nil, 71 | }, 72 | } 73 | 74 | for _, tc := range testCases { 75 | t.Run(tc.name, func(t *testing.T) { 76 | diff, err := NewCommitStats(tc.repo, tc.from, tc.to) 77 | require.NoError(t, err) 78 | 79 | result, err := diff.Eval(ctx, tc.row) 80 | require.NoError(t, err) 81 | 82 | require.EqualValues(t, tc.expected, result) 83 | }) 84 | } 85 | } 86 | 87 | func setupPool(t *testing.T) (*gitbase.RepositoryPool, func()) { 88 | t.Helper() 89 | 90 | path := fixtures.ByTag("worktree").One().Worktree().Root() 91 | pathLib := path + "-lib" 92 | pathRepo := filepath.Join(pathLib, "worktree") 93 | 94 | cleanup := func() { 95 | require.NoError(t, fixtures.Clean()) 96 | require.NoError(t, os.RemoveAll(pathLib)) 97 | } 98 | 99 | err := os.MkdirAll(pathLib, 0777) 100 | require.NoError(t, err) 101 | 102 | err = os.Rename(path, pathRepo) 103 | require.NoError(t, err) 104 | 105 | lib := plain.NewLibrary("plain", nil) 106 | loc, err := plain.NewLocation("location", osfs.New(pathLib), nil) 107 | require.NoError(t, err) 108 | lib.AddLocation(loc) 109 | 110 | pool := gitbase.NewRepositoryPool(cache.NewObjectLRUDefault(), lib) 111 | 112 | return pool, cleanup 113 | } 114 | -------------------------------------------------------------------------------- /internal/function/is_remote.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | 7 | "github.com/src-d/go-mysql-server/sql" 8 | "github.com/src-d/go-mysql-server/sql/expression" 9 | "gopkg.in/src-d/go-git.v4/plumbing" 10 | ) 11 | 12 | // IsRemote checks the given string is a remote reference. 13 | type IsRemote struct { 14 | expression.UnaryExpression 15 | } 16 | 17 | // NewIsRemote creates a new IsRemote function. 18 | func NewIsRemote(e sql.Expression) sql.Expression { 19 | return &IsRemote{expression.UnaryExpression{Child: e}} 20 | } 21 | 22 | // Eval implements the expression interface. 23 | func (f *IsRemote) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { 24 | span, ctx := ctx.Span("gitbase.IsRemote") 25 | defer span.Finish() 26 | 27 | val, err := f.Child.Eval(ctx, row) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | if val == nil { 33 | return false, nil 34 | } 35 | 36 | name, ok := val.(string) 37 | if !ok { 38 | return nil, sql.ErrInvalidType.New(reflect.TypeOf(val).String()) 39 | } 40 | 41 | return plumbing.ReferenceName(name).IsRemote(), nil 42 | } 43 | 44 | func (f IsRemote) String() string { 45 | return fmt.Sprintf("is_remote(%s)", f.Child) 46 | } 47 | 48 | // WithChildren implements the Expression interface. 49 | func (f IsRemote) WithChildren(children ...sql.Expression) (sql.Expression, error) { 50 | if len(children) != 1 { 51 | return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), 1) 52 | } 53 | return NewIsRemote(children[0]), nil 54 | } 55 | 56 | // Type implements the Expression interface. 57 | func (IsRemote) Type() sql.Type { 58 | return sql.Boolean 59 | } 60 | -------------------------------------------------------------------------------- /internal/function/is_remote_test.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | "github.com/src-d/go-mysql-server/sql" 9 | "github.com/src-d/go-mysql-server/sql/expression" 10 | ) 11 | 12 | func TestIsRemote(t *testing.T) { 13 | f := NewIsRemote(expression.NewGetField(0, sql.Text, "name", true)) 14 | 15 | testCases := []struct { 16 | name string 17 | row sql.Row 18 | expected bool 19 | err bool 20 | }{ 21 | {"null", sql.NewRow(nil), false, false}, 22 | {"not a branch", sql.NewRow("foo bar"), false, false}, 23 | {"not remote branch", sql.NewRow("refs/heads/foo"), false, false}, 24 | {"remote branch", sql.NewRow("refs/remotes/foo/bar"), true, false}, 25 | {"mismatched type", sql.NewRow(1), false, true}, 26 | } 27 | 28 | for _, tt := range testCases { 29 | t.Run(tt.name, func(t *testing.T) { 30 | require := require.New(t) 31 | 32 | session := sql.NewBaseSession() 33 | ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) 34 | 35 | val, err := f.Eval(ctx, tt.row) 36 | if tt.err { 37 | require.Error(err) 38 | require.True(sql.ErrInvalidType.Is(err)) 39 | } else { 40 | require.NoError(err) 41 | require.Equal(tt.expected, val) 42 | } 43 | }) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /internal/function/is_tag.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | 7 | "github.com/src-d/go-mysql-server/sql" 8 | "github.com/src-d/go-mysql-server/sql/expression" 9 | "gopkg.in/src-d/go-git.v4/plumbing" 10 | ) 11 | 12 | // IsTag checks the given string is a tag name. 13 | type IsTag struct { 14 | expression.UnaryExpression 15 | } 16 | 17 | // NewIsTag creates a new IsTag function. 18 | func NewIsTag(e sql.Expression) sql.Expression { 19 | return &IsTag{expression.UnaryExpression{Child: e}} 20 | } 21 | 22 | // Eval implements the expression interface. 23 | func (f *IsTag) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { 24 | span, ctx := ctx.Span("gitbase.IsTag") 25 | defer span.Finish() 26 | 27 | val, err := f.Child.Eval(ctx, row) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | if val == nil { 33 | return false, nil 34 | } 35 | 36 | name, ok := val.(string) 37 | if !ok { 38 | return nil, sql.ErrInvalidType.New(reflect.TypeOf(val).String()) 39 | } 40 | 41 | return plumbing.ReferenceName(name).IsTag(), nil 42 | } 43 | 44 | func (f IsTag) String() string { 45 | return fmt.Sprintf("is_tag(%s)", f.Child) 46 | } 47 | 48 | // WithChildren implements the Expression interface. 49 | func (f IsTag) WithChildren(children ...sql.Expression) (sql.Expression, error) { 50 | if len(children) != 1 { 51 | return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), 1) 52 | } 53 | return NewIsTag(children[0]), nil 54 | } 55 | 56 | // Type implements the Expression interface. 57 | func (IsTag) Type() sql.Type { 58 | return sql.Boolean 59 | } 60 | -------------------------------------------------------------------------------- /internal/function/is_tag_test.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | "github.com/src-d/go-mysql-server/sql" 9 | "github.com/src-d/go-mysql-server/sql/expression" 10 | ) 11 | 12 | func TestIsTag(t *testing.T) { 13 | f := NewIsTag(expression.NewGetField(0, sql.Text, "name", true)) 14 | 15 | testCases := []struct { 16 | name string 17 | row sql.Row 18 | expected bool 19 | err bool 20 | }{ 21 | {"null", sql.NewRow(nil), false, false}, 22 | {"not a ref name", sql.NewRow("foo bar"), false, false}, 23 | {"not a tag ref", sql.NewRow("refs/heads/v1.x"), false, false}, 24 | {"a tag", sql.NewRow("refs/tags/v1.0.0"), true, false}, 25 | {"mismatched type", sql.NewRow(1), false, true}, 26 | } 27 | 28 | for _, tt := range testCases { 29 | t.Run(tt.name, func(t *testing.T) { 30 | require := require.New(t) 31 | 32 | session := sql.NewBaseSession() 33 | ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) 34 | 35 | val, err := f.Eval(ctx, tt.row) 36 | if tt.err { 37 | require.Error(err) 38 | require.True(sql.ErrInvalidType.Is(err)) 39 | } else { 40 | require.NoError(err) 41 | require.Equal(tt.expected, val) 42 | } 43 | }) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /internal/function/is_vendor.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "fmt" 5 | 6 | enry "github.com/src-d/enry/v2" 7 | "github.com/src-d/go-mysql-server/sql" 8 | "github.com/src-d/go-mysql-server/sql/expression" 9 | ) 10 | 11 | // IsVendor reports whether files are vendored or not. 12 | type IsVendor struct { 13 | expression.UnaryExpression 14 | } 15 | 16 | // NewIsVendor creates a new IsVendor function. 17 | func NewIsVendor(filePath sql.Expression) sql.Expression { 18 | return &IsVendor{expression.UnaryExpression{Child: filePath}} 19 | } 20 | 21 | // Type implements the sql.Expression interface. 22 | func (v *IsVendor) Type() sql.Type { return sql.Boolean } 23 | 24 | // Eval implements the sql.Expression interface. 25 | func (v *IsVendor) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { 26 | span, ctx := ctx.Span("function.IsVendor") 27 | defer span.Finish() 28 | 29 | val, err := v.Child.Eval(ctx, row) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | if val == nil { 35 | return nil, nil 36 | } 37 | 38 | val, err = sql.Text.Convert(val) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | return enry.IsVendor(val.(string)), nil 44 | } 45 | 46 | func (v *IsVendor) String() string { 47 | return fmt.Sprintf("IS_VENDOR(%s)", v.Child) 48 | } 49 | 50 | // WithChildren implements the Expression interface. 51 | func (v IsVendor) WithChildren(children ...sql.Expression) (sql.Expression, error) { 52 | if len(children) != 1 { 53 | return nil, sql.ErrInvalidChildrenNumber.New(v, len(children), 1) 54 | } 55 | return NewIsVendor(children[0]), nil 56 | } 57 | -------------------------------------------------------------------------------- /internal/function/is_vendor_test.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/src-d/go-mysql-server/sql" 8 | "github.com/src-d/go-mysql-server/sql/expression" 9 | ) 10 | 11 | func TestIsVendor(t *testing.T) { 12 | testCases := []struct { 13 | name string 14 | path interface{} 15 | expected interface{} 16 | }{ 17 | { 18 | "non vendored path", 19 | "some/folder/foo.go", 20 | false, 21 | }, 22 | { 23 | "nil", 24 | nil, 25 | nil, 26 | }, 27 | { 28 | "vendored path", 29 | "vendor/foo.go", 30 | true, 31 | }, 32 | { 33 | "vendored (no root) path", 34 | "foo/bar/vendor/foo.go", 35 | true, 36 | }, 37 | } 38 | 39 | fn := NewIsVendor(expression.NewGetField(0, sql.Text, "x", true)) 40 | for _, tt := range testCases { 41 | t.Run(tt.name, func(t *testing.T) { 42 | result, err := fn.Eval(sql.NewEmptyContext(), sql.Row{tt.path}) 43 | require.NoError(t, err) 44 | require.Equal(t, tt.expected, result) 45 | }) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /internal/function/language.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "fmt" 5 | "hash/crc32" 6 | "os" 7 | "strconv" 8 | "sync" 9 | 10 | enry "github.com/src-d/enry/v2" 11 | "github.com/src-d/go-mysql-server/sql" 12 | ) 13 | 14 | const ( 15 | languageCacheSizeKey = "GITBASE_LANGUAGE_CACHE_SIZE" 16 | defaultLanguageCacheSize = 10000 17 | ) 18 | 19 | func languageCacheSize() int { 20 | v := os.Getenv(languageCacheSizeKey) 21 | size, err := strconv.Atoi(v) 22 | if err != nil || size <= 0 { 23 | size = defaultLanguageCacheSize 24 | } 25 | 26 | return size 27 | } 28 | 29 | var ( 30 | languageMut sync.Mutex 31 | languageCache sql.KeyValueCache 32 | ) 33 | 34 | func getLanguageCache(ctx *sql.Context) sql.KeyValueCache { 35 | languageMut.Lock() 36 | defer languageMut.Unlock() 37 | if languageCache == nil { 38 | // Dispose function is ignored because the cache will never be disposed 39 | // until the program dies. 40 | languageCache, _ = ctx.Memory.NewLRUCache(uint(languageCacheSize())) 41 | } 42 | 43 | return languageCache 44 | } 45 | 46 | // Language gets the language of a file given its path and 47 | // the optional content of the file. 48 | type Language struct { 49 | Left sql.Expression 50 | Right sql.Expression 51 | } 52 | 53 | // NewLanguage creates a new Language UDF. 54 | func NewLanguage(args ...sql.Expression) (sql.Expression, error) { 55 | var left, right sql.Expression 56 | switch len(args) { 57 | case 1: 58 | left = args[0] 59 | case 2: 60 | left = args[0] 61 | right = args[1] 62 | default: 63 | return nil, sql.ErrInvalidArgumentNumber.New("1 or 2", len(args)) 64 | } 65 | 66 | return &Language{left, right}, nil 67 | } 68 | 69 | // Resolved implements the Expression interface. 70 | func (f *Language) Resolved() bool { 71 | return f.Left.Resolved() && (f.Right == nil || f.Right.Resolved()) 72 | } 73 | 74 | func (f *Language) String() string { 75 | if f.Right == nil { 76 | return fmt.Sprintf("language(%s)", f.Left) 77 | } 78 | return fmt.Sprintf("language(%s, %s)", f.Left, f.Right) 79 | } 80 | 81 | // IsNullable implements the Expression interface. 82 | func (f *Language) IsNullable() bool { 83 | return f.Left.IsNullable() || (f.Right != nil && f.Right.IsNullable()) 84 | } 85 | 86 | // Type implements the Expression interface. 87 | func (Language) Type() sql.Type { 88 | return sql.Text 89 | } 90 | 91 | // WithChildren implements the Expression interface. 92 | func (f *Language) WithChildren(children ...sql.Expression) (sql.Expression, error) { 93 | expected := 1 94 | if f.Right != nil { 95 | expected = 2 96 | } 97 | 98 | if len(children) != expected { 99 | return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), expected) 100 | } 101 | 102 | return NewLanguage(children...) 103 | } 104 | 105 | // Eval implements the Expression interface. 106 | func (f *Language) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { 107 | span, ctx := ctx.Span("gitbase.Language") 108 | defer span.Finish() 109 | 110 | left, err := f.Left.Eval(ctx, row) 111 | if err != nil { 112 | return nil, err 113 | } 114 | 115 | if left == nil { 116 | return nil, nil 117 | } 118 | 119 | left, err = sql.Text.Convert(left) 120 | if err != nil { 121 | return nil, err 122 | } 123 | 124 | path := left.(string) 125 | var blob []byte 126 | 127 | if f.Right != nil { 128 | right, err := f.Right.Eval(ctx, row) 129 | if err != nil { 130 | return nil, err 131 | } 132 | 133 | if right == nil { 134 | return nil, nil 135 | } 136 | 137 | right, err = sql.Blob.Convert(right) 138 | if err != nil { 139 | return nil, err 140 | } 141 | 142 | blob = right.([]byte) 143 | } 144 | 145 | languageCache := getLanguageCache(ctx) 146 | 147 | var hash uint64 148 | if len(blob) > 0 { 149 | hash = languageHash(path, blob) 150 | value, err := languageCache.Get(hash) 151 | if err == nil { 152 | return value, nil 153 | } 154 | } 155 | 156 | lang := enry.GetLanguage(path, blob) 157 | if lang == "" { 158 | return nil, nil 159 | } 160 | 161 | if len(blob) > 0 { 162 | if err := languageCache.Put(hash, lang); err != nil { 163 | return nil, err 164 | } 165 | } 166 | 167 | return lang, nil 168 | } 169 | 170 | func languageHash(filename string, blob []byte) uint64 { 171 | fh := filenameHash(filename) 172 | bh := blobHash(blob) 173 | 174 | return uint64(fh)<<32 | uint64(bh) 175 | } 176 | 177 | func blobHash(blob []byte) uint32 { 178 | if len(blob) == 0 { 179 | return 0 180 | } 181 | 182 | return crc32.ChecksumIEEE(blob) 183 | } 184 | 185 | func filenameHash(filename string) uint32 { 186 | return crc32.ChecksumIEEE([]byte(filename)) 187 | } 188 | 189 | // Children implements the Expression interface. 190 | func (f *Language) Children() []sql.Expression { 191 | if f.Right == nil { 192 | return []sql.Expression{f.Left} 193 | } 194 | 195 | return []sql.Expression{f.Left, f.Right} 196 | } 197 | -------------------------------------------------------------------------------- /internal/function/language_test.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | errors "gopkg.in/src-d/go-errors.v1" 8 | "github.com/src-d/go-mysql-server/sql" 9 | "github.com/src-d/go-mysql-server/sql/expression" 10 | ) 11 | 12 | func TestLanguage(t *testing.T) { 13 | testCases := []struct { 14 | name string 15 | row sql.Row 16 | expected interface{} 17 | err *errors.Kind 18 | }{ 19 | {"left is null", sql.NewRow(nil), nil, nil}, 20 | {"right is null", sql.NewRow("foo", nil), nil, nil}, 21 | {"both are null", sql.NewRow(nil, nil), nil, nil}, 22 | {"only path is given", sql.NewRow("foo.rb"), "Ruby", nil}, 23 | {"only path is given", sql.NewRow("foo.foobar"), nil, nil}, 24 | {"too many args given", sql.NewRow("foo.rb", "bar", "baz"), nil, sql.ErrInvalidArgumentNumber}, 25 | {"path and blob are given", sql.NewRow("foo", "#!/usr/bin/env python\n\nprint 'foo'"), "Python", nil}, 26 | {"invalid blob type given", sql.NewRow("foo", 5), nil, sql.ErrInvalidType}, 27 | } 28 | 29 | for _, tt := range testCases { 30 | t.Run(tt.name, func(t *testing.T) { 31 | require := require.New(t) 32 | ctx := sql.NewEmptyContext() 33 | 34 | var args = make([]sql.Expression, len(tt.row)) 35 | for i := range tt.row { 36 | args[i] = expression.NewGetField(i, sql.Text, "", false) 37 | } 38 | 39 | f, err := NewLanguage(args...) 40 | if err == nil { 41 | var val interface{} 42 | val, err = f.Eval(ctx, tt.row) 43 | if tt.err == nil { 44 | require.NoError(err) 45 | require.Equal(tt.expected, val) 46 | } 47 | } 48 | 49 | if tt.err != nil { 50 | require.Error(err) 51 | require.True(tt.err.Is(err)) 52 | } 53 | }) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /internal/function/loc.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/hhatto/gocloc" 9 | "github.com/src-d/enry/v2" 10 | "github.com/src-d/go-mysql-server/sql" 11 | ) 12 | 13 | var languages = gocloc.NewDefinedLanguages() 14 | 15 | var errEmptyInputValues = errors.New("empty input values") 16 | 17 | // LOC is a function that returns the count of different types of lines of code. 18 | type LOC struct { 19 | Left sql.Expression 20 | Right sql.Expression 21 | } 22 | 23 | // NewLOC creates a new LOC UDF. 24 | func NewLOC(args ...sql.Expression) (sql.Expression, error) { 25 | if len(args) != 2 { 26 | return nil, sql.ErrInvalidArgumentNumber.New("2", len(args)) 27 | } 28 | 29 | return &LOC{args[0], args[1]}, nil 30 | } 31 | 32 | // Resolved implements the Expression interface. 33 | func (f *LOC) Resolved() bool { 34 | return f.Left.Resolved() && f.Right.Resolved() 35 | } 36 | 37 | func (f *LOC) String() string { 38 | return fmt.Sprintf("loc(%s, %s)", f.Left, f.Right) 39 | } 40 | 41 | // IsNullable implements the Expression interface. 42 | func (f *LOC) IsNullable() bool { 43 | return f.Left.IsNullable() || f.Right.IsNullable() 44 | } 45 | 46 | // Type implements the Expression interface. 47 | func (LOC) Type() sql.Type { 48 | return sql.JSON 49 | } 50 | 51 | // WithChildren implements the Expression interface. 52 | func (f *LOC) WithChildren(children ...sql.Expression) (sql.Expression, error) { 53 | return NewLOC(children...) 54 | } 55 | 56 | // LocFile is the result of the LOC function for each file. 57 | type LocFile struct { 58 | Code int32 `json:"Code"` 59 | Comments int32 `json:"Comment"` 60 | Blanks int32 `json:"Blank"` 61 | Name string `json:"Name"` 62 | Lang string `json:"Language"` 63 | } 64 | 65 | // Eval implements the Expression interface. 66 | func (f *LOC) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { 67 | span, ctx := ctx.Span("gitbase.LOC") 68 | defer span.Finish() 69 | path, blob, err := f.getInputValues(ctx, row) 70 | if err != nil { 71 | if err == errEmptyInputValues { 72 | return nil, nil 73 | } 74 | 75 | return nil, err 76 | } 77 | 78 | lang, err := getLanguage(ctx, path, blob) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | if lang == "" || languages.Langs[lang] == nil { 84 | return nil, nil 85 | } 86 | 87 | file := gocloc.AnalyzeReader( 88 | path, 89 | languages.Langs[lang], 90 | bytes.NewReader(blob), &gocloc.ClocOptions{}, 91 | ) 92 | 93 | return LocFile{ 94 | Code: file.Code, 95 | Comments: file.Comments, 96 | Blanks: file.Blanks, 97 | Name: file.Name, 98 | Lang: file.Lang, 99 | }, nil 100 | } 101 | 102 | func (f *LOC) getInputValues(ctx *sql.Context, row sql.Row) (string, []byte, error) { 103 | left, err := f.Left.Eval(ctx, row) 104 | if err != nil { 105 | return "", nil, err 106 | } 107 | 108 | left, err = sql.Text.Convert(left) 109 | if err != nil { 110 | return "", nil, err 111 | } 112 | 113 | right, err := f.Right.Eval(ctx, row) 114 | if err != nil { 115 | return "", nil, err 116 | } 117 | 118 | right, err = sql.Blob.Convert(right) 119 | if err != nil { 120 | return "", nil, err 121 | } 122 | 123 | if right == nil { 124 | return "", nil, errEmptyInputValues 125 | } 126 | 127 | path, ok := left.(string) 128 | if !ok { 129 | return "", nil, errEmptyInputValues 130 | } 131 | 132 | blob, ok := right.([]byte) 133 | 134 | if !ok { 135 | return "", nil, errEmptyInputValues 136 | } 137 | 138 | if len(blob) == 0 || len(path) == 0 { 139 | return "", nil, errEmptyInputValues 140 | } 141 | 142 | return path, blob, nil 143 | } 144 | 145 | func getLanguage(ctx *sql.Context, path string, blob []byte) (string, error) { 146 | hash := languageHash(path, blob) 147 | 148 | languageCache := getLanguageCache(ctx) 149 | value, err := languageCache.Get(hash) 150 | if err == nil { 151 | return value.(string), nil 152 | } 153 | 154 | lang := enry.GetLanguage(path, blob) 155 | if len(blob) > 0 { 156 | if err := languageCache.Put(hash, lang); err != nil { 157 | return "", err 158 | } 159 | } 160 | 161 | return lang, nil 162 | } 163 | 164 | // Children implements the Expression interface. 165 | func (f *LOC) Children() []sql.Expression { 166 | if f.Right == nil { 167 | return []sql.Expression{f.Left} 168 | } 169 | 170 | return []sql.Expression{f.Left, f.Right} 171 | } 172 | -------------------------------------------------------------------------------- /internal/function/loc_test.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/src-d/go-mysql-server/sql" 7 | "github.com/src-d/go-mysql-server/sql/expression" 8 | "github.com/stretchr/testify/require" 9 | "gopkg.in/src-d/go-errors.v1" 10 | ) 11 | 12 | func TestLoc(t *testing.T) { 13 | testCases := []struct { 14 | name string 15 | row sql.Row 16 | expected interface{} 17 | err *errors.Kind 18 | }{ 19 | {"left is null", sql.NewRow(nil), nil, nil}, 20 | {"both are null", sql.NewRow(nil, nil), nil, nil}, 21 | {"too few args given", sql.NewRow("foo.foobar"), nil, nil}, 22 | {"too many args given", sql.NewRow("foo.rb", "bar", "baz"), nil, sql.ErrInvalidArgumentNumber}, 23 | {"invalid blob type given", sql.NewRow("foo", 5), nil, sql.ErrInvalidType}, 24 | {"path and blob are given", sql.NewRow("foo", "#!/usr/bin/env python\n\nprint 'foo'"), LocFile{ 25 | Code: 2, Comments: 0, Blanks: 1, Name: "foo", Lang: "Python", 26 | }, nil}, 27 | } 28 | 29 | for _, tt := range testCases { 30 | t.Run(tt.name, func(t *testing.T) { 31 | require := require.New(t) 32 | ctx := sql.NewEmptyContext() 33 | 34 | var args = make([]sql.Expression, len(tt.row)) 35 | for i := range tt.row { 36 | args[i] = expression.NewGetField(i, sql.Text, "", false) 37 | } 38 | 39 | f, err := NewLOC(args...) 40 | if err == nil { 41 | var val interface{} 42 | val, err = f.Eval(ctx, tt.row) 43 | if tt.err == nil { 44 | require.NoError(err) 45 | require.Equal(tt.expected, val) 46 | } 47 | } 48 | 49 | if tt.err != nil { 50 | require.Error(err) 51 | require.True(tt.err.Is(err)) 52 | } 53 | }) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /internal/function/registry.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import "github.com/src-d/go-mysql-server/sql" 4 | 5 | // Functions for gitbase queries. 6 | var Functions = []sql.Function{ 7 | sql.FunctionN{Name: "commit_stats", Fn: NewCommitStats}, 8 | sql.FunctionN{Name: "commit_file_stats", Fn: NewCommitFileStats}, 9 | sql.Function1{Name: "is_tag", Fn: NewIsTag}, 10 | sql.Function1{Name: "is_remote", Fn: NewIsRemote}, 11 | sql.FunctionN{Name: "language", Fn: NewLanguage}, 12 | sql.FunctionN{Name: "loc", Fn: NewLOC}, 13 | sql.FunctionN{Name: "uast", Fn: NewUAST}, 14 | sql.Function3{Name: "uast_mode", Fn: NewUASTMode}, 15 | sql.Function2{Name: "uast_xpath", Fn: NewUASTXPath}, 16 | sql.Function2{Name: "uast_extract", Fn: NewUASTExtract}, 17 | sql.Function1{Name: "uast_children", Fn: NewUASTChildren}, 18 | sql.Function1{Name: "uast_imports", Fn: NewUASTImports}, 19 | sql.Function1{Name: "is_vendor", Fn: NewIsVendor}, 20 | sql.Function3{Name: "blame", Fn: NewBlame}, 21 | } 22 | -------------------------------------------------------------------------------- /internal/function/uast_utils.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "hash" 7 | "hash/crc64" 8 | 9 | "github.com/bblfsh/go-client/v4/tools" 10 | "github.com/bblfsh/sdk/v3/uast/nodes/nodesproto" 11 | 12 | bblfsh "github.com/bblfsh/go-client/v4" 13 | "github.com/bblfsh/sdk/v3/uast/nodes" 14 | "github.com/sirupsen/logrus" 15 | "github.com/src-d/gitbase" 16 | "github.com/src-d/go-mysql-server/sql" 17 | errors "gopkg.in/src-d/go-errors.v1" 18 | ) 19 | 20 | var ( 21 | // ErrParseBlob is returned when the blob can't be parsed with bblfsh. 22 | ErrParseBlob = errors.NewKind("unable to parse the given blob using bblfsh: %s") 23 | 24 | // ErrUnmarshalUAST is returned when an error arises unmarshaling UASTs. 25 | ErrUnmarshalUAST = errors.NewKind("error unmarshaling UAST: %s") 26 | 27 | // ErrMarshalUAST is returned when an error arises marshaling UASTs. 28 | ErrMarshalUAST = errors.NewKind("error marshaling uast node: %s") 29 | ) 30 | 31 | func exprToString( 32 | ctx *sql.Context, 33 | e sql.Expression, 34 | r sql.Row, 35 | ) (string, error) { 36 | if e == nil { 37 | return "", nil 38 | } 39 | 40 | x, err := e.Eval(ctx, r) 41 | if err != nil { 42 | return "", err 43 | } 44 | 45 | if x == nil { 46 | return "", nil 47 | } 48 | 49 | x, err = sql.Text.Convert(x) 50 | if err != nil { 51 | return "", err 52 | } 53 | 54 | return x.(string), nil 55 | } 56 | 57 | var crcTable = crc64.MakeTable(crc64.ISO) 58 | 59 | func newHash() hash.Hash64 { 60 | return crc64.New(crcTable) 61 | } 62 | 63 | func computeKey(h hash.Hash64, mode, lang string, blob []byte) (uint64, error) { 64 | h.Reset() 65 | if err := writeToHash(h, [][]byte{ 66 | []byte(mode), 67 | []byte(lang), 68 | blob, 69 | }); err != nil { 70 | return 0, err 71 | } 72 | 73 | return h.Sum64(), nil 74 | } 75 | 76 | func writeToHash(h hash.Hash, elements [][]byte) error { 77 | for _, e := range elements { 78 | n, err := h.Write(e) 79 | if err != nil { 80 | return err 81 | } 82 | 83 | if n != len(e) { 84 | return fmt.Errorf("cache key hash: " + 85 | "couldn't write all the content") 86 | } 87 | } 88 | 89 | return nil 90 | } 91 | 92 | func getUASTFromBblfsh(ctx *sql.Context, 93 | blob []byte, 94 | lang, xpath string, 95 | mode bblfsh.Mode, 96 | ) (nodes.Node, error) { 97 | session, ok := ctx.Session.(*gitbase.Session) 98 | if !ok { 99 | return nil, gitbase.ErrInvalidGitbaseSession.New(ctx.Session) 100 | } 101 | 102 | client, err := session.BblfshClient() 103 | if err != nil { 104 | return nil, err 105 | } 106 | 107 | // If we have a language we must check if it's supported. If we don't, bblfsh 108 | // is the one that will have to identify the language. 109 | if lang != "" { 110 | ok, err = client.IsLanguageSupported(ctx, lang) 111 | if err != nil { 112 | return nil, err 113 | } 114 | 115 | if !ok { 116 | return nil, ErrParseBlob.New( 117 | fmt.Errorf("unsupported language %q", lang)) 118 | } 119 | } 120 | 121 | node, _, err := client.ParseWithMode(ctx, mode, lang, blob) 122 | if err != nil { 123 | err := ErrParseBlob.New(err) 124 | logrus.Warn(err) 125 | ctx.Warn(0, err.Error()) 126 | return nil, err 127 | } 128 | 129 | return node, nil 130 | } 131 | 132 | func applyXpath(n nodes.Node, query string) (nodes.Array, error) { 133 | var filtered nodes.Array 134 | it, err := tools.Filter(n, query) 135 | if err != nil { 136 | return nil, err 137 | } 138 | 139 | for n := range tools.Iterate(it) { 140 | filtered = append(filtered, n) 141 | } 142 | 143 | return filtered, nil 144 | } 145 | 146 | func marshalNodes(arr nodes.Array) (interface{}, error) { 147 | if len(arr) == 0 { 148 | return nil, nil 149 | } 150 | 151 | var buf bytes.Buffer 152 | if err := nodesproto.WriteTo(&buf, arr); err != nil { 153 | return nil, err 154 | } 155 | 156 | return buf.Bytes(), nil 157 | } 158 | 159 | func getNodes(data interface{}) (nodes.Array, error) { 160 | if data == nil { 161 | return nil, nil 162 | } 163 | 164 | raw, ok := data.([]byte) 165 | if !ok { 166 | return nil, ErrUnmarshalUAST.New("wrong underlying UAST format") 167 | } 168 | 169 | return unmarshalNodes(raw) 170 | } 171 | 172 | func unmarshalNodes(data []byte) (nodes.Array, error) { 173 | if len(data) == 0 { 174 | return nil, nil 175 | } 176 | 177 | buf := bytes.NewReader(data) 178 | n, err := nodesproto.ReadTree(buf) 179 | if err != nil { 180 | return nil, err 181 | } 182 | 183 | if n.Kind() != nodes.KindArray { 184 | return nil, fmt.Errorf("unmarshal: wrong kind of node found %q, expected %q", 185 | n.Kind(), nodes.KindArray.String()) 186 | } 187 | 188 | return n.(nodes.Array), nil 189 | } 190 | -------------------------------------------------------------------------------- /packfiles_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | fixtures "github.com/src-d/go-git-fixtures" 9 | "github.com/stretchr/testify/require" 10 | "gopkg.in/src-d/go-git.v4/plumbing" 11 | "gopkg.in/src-d/go-git.v4/plumbing/cache" 12 | ) 13 | 14 | var ( 15 | testSivaFilePath = filepath.Join("_testdata", "fff7062de8474d10a67d417ccea87ba6f58ca81d.siva") 16 | testSivaRepoID = "015dcc49-9049-b00c-ba72-b6f5fa98cbe7" 17 | ) 18 | 19 | func TestRepositoryPackfiles(t *testing.T) { 20 | require := require.New(t) 21 | 22 | lib, pool, err := newMultiPool() 23 | require.NoError(err) 24 | 25 | cwd, err := os.Getwd() 26 | require.NoError(err) 27 | p := filepath.Join(cwd, testSivaFilePath) 28 | 29 | err = lib.AddSiva(p, nil) 30 | require.NoError(err) 31 | 32 | repo, err := pool.GetRepo(testSivaRepoID) 33 | require.NoError(err) 34 | 35 | f, err := repo.FS() 36 | require.NoError(err) 37 | 38 | fs, packfiles, err := repositoryPackfiles(f) 39 | 40 | require.NoError(err) 41 | require.Equal([]plumbing.Hash{ 42 | plumbing.NewHash("433e5205f6e26099e7d34ba5e5306f69e4cef12b"), 43 | plumbing.NewHash("5d2ce6a45cb07803f9b0c8040e730f5715fc7144"), 44 | }, packfiles) 45 | require.NotNil(fs) 46 | } 47 | 48 | func TestRepositoryPackfilesNoBare(t *testing.T) { 49 | require := require.New(t) 50 | 51 | fs := fixtures.ByTag("worktree").One().Worktree() 52 | 53 | dotgit, packfiles, err := repositoryPackfiles(fs) 54 | require.NoError(err) 55 | require.Equal([]plumbing.Hash{ 56 | plumbing.NewHash("323a4b6b5de684f9966953a043bc800154e5dbfa"), 57 | }, packfiles) 58 | 59 | require.NoError(dotgit.Close()) 60 | } 61 | 62 | func TestGetUnpackedObject(t *testing.T) { 63 | require := require.New(t) 64 | 65 | fs := fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One().Worktree() 66 | path := fs.Root() 67 | 68 | lib, err := newMultiLibrary() 69 | require.NoError(err) 70 | pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib) 71 | require.NoError(lib.AddPlain(path, path, nil)) 72 | 73 | r, err := pool.GetRepo(path) 74 | require.NoError(err) 75 | 76 | obj, err := getUnpackedObject(r, plumbing.NewHash("3bf5d30ad4f23cf517676fee232e3bcb8537c1d0")) 77 | require.NoError(err) 78 | require.NotNil(obj) 79 | require.NoError(r.Close()) 80 | } 81 | 82 | func TestRepositoryIndex(t *testing.T) { 83 | lib, pool, err := newMultiPool() 84 | require.NoError(t, err) 85 | 86 | cwd, err := os.Getwd() 87 | require.NoError(t, err) 88 | p := filepath.Join(cwd, testSivaFilePath) 89 | 90 | err = lib.AddSiva(p, nil) 91 | require.NoError(t, err) 92 | 93 | repo, err := pool.GetRepo(testSivaRepoID) 94 | require.NoError(t, err) 95 | 96 | idx, err := newRepositoryIndex(repo) 97 | require.NoError(t, err) 98 | 99 | testCases := []struct { 100 | hash string 101 | offset int64 102 | packfile string 103 | }{ 104 | { 105 | "52c853392c25d3a670446641f4b44b22770b3bbe", 106 | 3046713, 107 | "5d2ce6a45cb07803f9b0c8040e730f5715fc7144", 108 | }, 109 | { 110 | "aa7ef7dafd292737ed493b7d74c0abfa761344f4", 111 | 3046902, 112 | "5d2ce6a45cb07803f9b0c8040e730f5715fc7144", 113 | }, 114 | } 115 | 116 | for _, tt := range testCases { 117 | t.Run(tt.hash, func(t *testing.T) { 118 | offset, packfile, err := idx.find(plumbing.NewHash(tt.hash)) 119 | require.NoError(t, err) 120 | require.Equal(t, tt.offset, offset) 121 | require.Equal(t, tt.packfile, packfile.String()) 122 | }) 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /path_utils.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "path/filepath" 5 | "regexp" 6 | "strings" 7 | 8 | git "gopkg.in/src-d/go-git.v4" 9 | ) 10 | 11 | // RegMatchChars matches a string with a glob expression inside. 12 | var RegMatchChars = regexp.MustCompile(`(^|[^\\])([*[?])`) 13 | 14 | // StripPrefix removes the root path from the given path. Root may be a glob. 15 | // The returned string has all backslashes replaced with slashes. 16 | func StripPrefix(root, path string) (string, error) { 17 | var err error 18 | root, err = filepath.Abs(cleanGlob(root)) 19 | if err != nil { 20 | return "", err 21 | } 22 | 23 | if !strings.HasSuffix(root, "/") { 24 | root += string(filepath.Separator) 25 | } 26 | 27 | path, err = filepath.Abs(path) 28 | if err != nil { 29 | return "", err 30 | } 31 | 32 | return strings.TrimPrefix(filepath.ToSlash(path), root), nil 33 | } 34 | 35 | // cleanGlob removes all the parts of a glob that are not fixed. It also 36 | // converts all slashes or backslashes to /. 37 | func cleanGlob(pattern string) string { 38 | pattern = filepath.ToSlash(pattern) 39 | var parts []string 40 | for _, part := range strings.Split(pattern, "/") { 41 | if strings.ContainsAny(part, "*?[\\") { 42 | break 43 | } 44 | 45 | parts = append(parts, part) 46 | } 47 | 48 | return strings.Join(parts, "/") 49 | } 50 | 51 | // PatternMatches returns the paths matched and any error found. 52 | func PatternMatches(pattern string) ([]string, error) { 53 | abs, err := filepath.Abs(pattern) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | matches, err := filepath.Glob(abs) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | return removeDsStore(matches), nil 64 | } 65 | 66 | func removeDsStore(matches []string) []string { 67 | var result []string 68 | for _, m := range matches { 69 | if filepath.Base(m) != ".DS_Store" { 70 | result = append(result, m) 71 | } 72 | } 73 | return result 74 | } 75 | 76 | // IsGitRepo checks that the given path is a git repository. 77 | func IsGitRepo(path string) (bool, error) { 78 | if _, err := git.PlainOpen(path); err != nil { 79 | if git.ErrRepositoryNotExists == err { 80 | return false, nil 81 | } 82 | 83 | return false, err 84 | } 85 | 86 | return true, nil 87 | } 88 | 89 | //IsSivaFile checks that the given file is a siva file. 90 | func IsSivaFile(file string) bool { 91 | return strings.HasSuffix(file, ".siva") 92 | } 93 | -------------------------------------------------------------------------------- /path_utils_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | fixtures "github.com/src-d/go-git-fixtures" 10 | ) 11 | 12 | func TestPatternMatches(t *testing.T) { 13 | wd, err := os.Getwd() 14 | require.NoError(t, err) 15 | 16 | testCases := []struct { 17 | path string 18 | expected []string 19 | }{ 20 | {"cmd", []string{ 21 | filepath.Join(wd, "cmd"), 22 | }}, 23 | {"cmd/*", []string{ 24 | filepath.Join(wd, "cmd/gitbase"), 25 | }}, 26 | {"cmd/gitbase/*", []string{ 27 | filepath.Join(wd, "cmd/gitbase/command"), 28 | filepath.Join(wd, "cmd/gitbase/main.go"), 29 | }}, 30 | {"cmd/../cmd/gitbase/*", []string{ 31 | filepath.Join(wd, "cmd/gitbase/command"), 32 | filepath.Join(wd, "cmd/gitbase/main.go"), 33 | }}, 34 | } 35 | 36 | for _, test := range testCases { 37 | t.Run(test.path, func(t *testing.T) { 38 | files, err := PatternMatches(test.path) 39 | require.NoError(t, err) 40 | require.Exactly(t, test.expected, files) 41 | }) 42 | } 43 | } 44 | 45 | func TestIsGitRepo(t *testing.T) { 46 | var require = require.New(t) 47 | 48 | ok, err := IsGitRepo("/do/not/exist") 49 | require.NoError(err) 50 | require.False(ok) 51 | 52 | path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() 53 | ok, err = IsGitRepo(path) 54 | require.NoError(err) 55 | require.True(ok) 56 | } 57 | 58 | func TestIsSivaFile(t *testing.T) { 59 | var require = require.New(t) 60 | 61 | require.True(IsSivaFile("is.siva")) 62 | require.False(IsSivaFile("not-siva")) 63 | } 64 | 65 | func TestStripPrefix(t *testing.T) { 66 | testCases := []struct { 67 | root string 68 | path string 69 | expected string 70 | }{ 71 | { 72 | "_testdata/*", 73 | "_testdata/05893125684f2d3943cd84a7ab2b75e53668fba1.siva", 74 | "05893125684f2d3943cd84a7ab2b75e53668fba1.siva", 75 | }, 76 | { 77 | "_testdata/*", 78 | "_testdata/foo/05893125684f2d3943cd84a7ab2b75e53668fba1.siva", 79 | "foo/05893125684f2d3943cd84a7ab2b75e53668fba1.siva", 80 | }, 81 | } 82 | 83 | for _, tt := range testCases { 84 | t.Run(tt.path, func(t *testing.T) { 85 | output, err := StripPrefix(tt.root, tt.path) 86 | require.NoError(t, err) 87 | require.Equal(t, tt.expected, output) 88 | }) 89 | } 90 | } 91 | 92 | func TestCleanGlob(t *testing.T) { 93 | testCases := []struct { 94 | pattern string 95 | expected string 96 | }{ 97 | {"../../../_testdata/?epositories", "../../../_testdata"}, 98 | {"../../../_testdata/**/repositories", "../../../_testdata"}, 99 | {"../../../_testdata/*/repositories", "../../../_testdata"}, 100 | {"../../../_testdata/*", "../../../_testdata"}, 101 | {"../../../_testdata/\\*/foo", "../../../_testdata"}, 102 | {"../../../_testdata/[a-z]/foo", "../../../_testdata"}, 103 | } 104 | 105 | for _, tt := range testCases { 106 | t.Run(tt.pattern, func(t *testing.T) { 107 | output := cleanGlob(tt.pattern) 108 | require.Equal(t, tt.expected, output) 109 | }) 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | - [ ] I updated the documentation explaining the new behavior if any. 4 | - [ ] I updated CHANGELOG.md file adding the new feature or bug fix. 5 | - [ ] I updated go-mysql-server using `make upgrade` command if applicable. 6 | - [ ] I added or updated examples if applicable. 7 | - [ ] I checked that changes on schema are reflected into the documentation, if applicable. -------------------------------------------------------------------------------- /ref_commits_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/src-d/go-mysql-server/sql" 7 | "github.com/src-d/go-mysql-server/sql/expression" 8 | "github.com/stretchr/testify/require" 9 | "gopkg.in/src-d/go-git.v4/plumbing" 10 | ) 11 | 12 | func TestRefCommitsRowIter(t *testing.T) { 13 | require := require.New(t) 14 | ctx, _, cleanup := setup(t) 15 | defer cleanup() 16 | 17 | rows, err := tableToRows(ctx, newRefCommitsTable(poolFromCtx(t, ctx))) 18 | require.NoError(err) 19 | 20 | for i, row := range rows { 21 | // remove repository ids 22 | rows[i] = row[1:] 23 | } 24 | 25 | expected := []sql.Row{ 26 | {"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "HEAD", int64(0)}, 27 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "HEAD", int64(1)}, 28 | {"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "HEAD", int64(2)}, 29 | {"1669dce138d9b841a518c64b10914d88f5e488ea", "HEAD", int64(3)}, 30 | {"35e85108805c84807bc66a02d91535e1e24b38b9", "HEAD", int64(4)}, 31 | {"b029517f6300c2da0f4b651b8642506cd6aaf45d", "HEAD", int64(5)}, 32 | {"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "HEAD", int64(4)}, 33 | {"b8e471f58bcbca63b07bda20e428190409c2db47", "HEAD", int64(5)}, 34 | 35 | {"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "refs/heads/master", int64(0)}, 36 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "refs/heads/master", int64(1)}, 37 | {"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "refs/heads/master", int64(2)}, 38 | {"1669dce138d9b841a518c64b10914d88f5e488ea", "refs/heads/master", int64(3)}, 39 | {"35e85108805c84807bc66a02d91535e1e24b38b9", "refs/heads/master", int64(4)}, 40 | {"b029517f6300c2da0f4b651b8642506cd6aaf45d", "refs/heads/master", int64(5)}, 41 | {"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "refs/heads/master", int64(4)}, 42 | {"b8e471f58bcbca63b07bda20e428190409c2db47", "refs/heads/master", int64(5)}, 43 | 44 | {"e8d3ffab552895c19b9fcf7aa264d277cde33881", "refs/remotes/origin/branch", int64(0)}, 45 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "refs/remotes/origin/branch", int64(1)}, 46 | {"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "refs/remotes/origin/branch", int64(2)}, 47 | {"1669dce138d9b841a518c64b10914d88f5e488ea", "refs/remotes/origin/branch", int64(3)}, 48 | {"35e85108805c84807bc66a02d91535e1e24b38b9", "refs/remotes/origin/branch", int64(4)}, 49 | {"b029517f6300c2da0f4b651b8642506cd6aaf45d", "refs/remotes/origin/branch", int64(5)}, 50 | {"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "refs/remotes/origin/branch", int64(4)}, 51 | {"b8e471f58bcbca63b07bda20e428190409c2db47", "refs/remotes/origin/branch", int64(5)}, 52 | 53 | {"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "refs/remotes/origin/master", int64(0)}, 54 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "refs/remotes/origin/master", int64(1)}, 55 | {"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "refs/remotes/origin/master", int64(2)}, 56 | {"1669dce138d9b841a518c64b10914d88f5e488ea", "refs/remotes/origin/master", int64(3)}, 57 | {"35e85108805c84807bc66a02d91535e1e24b38b9", "refs/remotes/origin/master", int64(4)}, 58 | {"b029517f6300c2da0f4b651b8642506cd6aaf45d", "refs/remotes/origin/master", int64(5)}, 59 | {"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "refs/remotes/origin/master", int64(4)}, 60 | {"b8e471f58bcbca63b07bda20e428190409c2db47", "refs/remotes/origin/master", int64(5)}, 61 | } 62 | 63 | require.Equal(expected, rows) 64 | } 65 | 66 | func TestRefCommitsPushdown(t *testing.T) { 67 | ctx, _, cleanup := setup(t) 68 | defer cleanup() 69 | 70 | table := new(refCommitsTable) 71 | testCases := []struct { 72 | name string 73 | filters []sql.Expression 74 | expected []sql.Row 75 | }{ 76 | { 77 | "ref filter", 78 | []sql.Expression{ 79 | expression.NewEquals( 80 | expression.NewGetFieldWithTable(0, sql.Text, RefCommitsTableName, "ref_name", false), 81 | expression.NewLiteral("HEAD", sql.Text), 82 | ), 83 | }, 84 | []sql.Row{ 85 | {"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "HEAD", int64(0)}, 86 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "HEAD", int64(1)}, 87 | {"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "HEAD", int64(2)}, 88 | {"1669dce138d9b841a518c64b10914d88f5e488ea", "HEAD", int64(3)}, 89 | {"35e85108805c84807bc66a02d91535e1e24b38b9", "HEAD", int64(4)}, 90 | {"b029517f6300c2da0f4b651b8642506cd6aaf45d", "HEAD", int64(5)}, 91 | {"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "HEAD", int64(4)}, 92 | {"b8e471f58bcbca63b07bda20e428190409c2db47", "HEAD", int64(5)}, 93 | }, 94 | }, 95 | { 96 | "ref filter", 97 | []sql.Expression{ 98 | expression.NewEquals( 99 | expression.NewGetFieldWithTable(1, sql.Text, RefCommitsTableName, "commit_hash", false), 100 | expression.NewLiteral("918c48b83bd081e863dbe1b80f8998f058cd8294", sql.Text), 101 | ), 102 | }, 103 | []sql.Row{ 104 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "HEAD", int64(1)}, 105 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "refs/heads/master", int64(1)}, 106 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "refs/remotes/origin/branch", int64(1)}, 107 | {"918c48b83bd081e863dbe1b80f8998f058cd8294", "refs/remotes/origin/master", int64(1)}, 108 | }, 109 | }, 110 | } 111 | 112 | for _, tt := range testCases { 113 | t.Run(tt.name, func(t *testing.T) { 114 | require := require.New(t) 115 | tbl := table.WithFilters(tt.filters) 116 | 117 | rows, err := tableToRows(ctx, tbl) 118 | require.NoError(err) 119 | 120 | for i, row := range rows { 121 | // remove blob content and blob size for better diffs 122 | // and repository_ids 123 | rows[i] = row[1:] 124 | } 125 | 126 | require.Equal(tt.expected, rows) 127 | }) 128 | } 129 | } 130 | 131 | func TestRefCommitsIndexKeyValueIter(t *testing.T) { 132 | require := require.New(t) 133 | ctx, _, cleanup := setup(t) 134 | defer cleanup() 135 | 136 | table := new(refCommitsTable) 137 | iter, err := table.IndexKeyValues(ctx, []string{"ref_name", "commit_hash"}) 138 | require.NoError(err) 139 | 140 | rows, err := tableToRows(ctx, table) 141 | require.NoError(err) 142 | 143 | var expected []keyValue 144 | for _, row := range rows { 145 | var kv keyValue 146 | kv.key = assertEncodeRefCommitsRow(t, row) 147 | kv.values = append(kv.values, row[2], row[1]) 148 | expected = append(expected, kv) 149 | } 150 | 151 | assertIndexKeyValueIter(t, iter, expected) 152 | } 153 | 154 | func assertEncodeRefCommitsRow(t *testing.T, row sql.Row) []byte { 155 | t.Helper() 156 | k, err := new(refCommitsRowKeyMapper).fromRow(row) 157 | require.NoError(t, err) 158 | return k 159 | } 160 | 161 | func TestRefCommitsIndex(t *testing.T) { 162 | testTableIndex( 163 | t, 164 | new(refCommitsTable), 165 | []sql.Expression{expression.NewEquals( 166 | expression.NewGetField(2, sql.Text, "ref_name", false), 167 | expression.NewLiteral("HEAD", sql.Text), 168 | )}, 169 | ) 170 | } 171 | 172 | func TestRefCommitsRowKeyMapper(t *testing.T) { 173 | require := require.New(t) 174 | row := sql.Row{"repo1", plumbing.ZeroHash.String(), "ref_name", int64(1)} 175 | mapper := new(refCommitsRowKeyMapper) 176 | 177 | k, err := mapper.fromRow(row) 178 | require.NoError(err) 179 | 180 | row2, err := mapper.toRow(k) 181 | require.NoError(err) 182 | 183 | require.Equal(row, row2) 184 | } 185 | 186 | func TestRefCommitsIndexIterClosed(t *testing.T) { 187 | testTableIndexIterClosed(t, new(refCommitsTable)) 188 | } 189 | 190 | func TestRefCommitsIterClosed(t *testing.T) { 191 | testTableIterClosed(t, new(refCommitsTable)) 192 | } 193 | -------------------------------------------------------------------------------- /references_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/src-d/go-mysql-server/sql" 7 | "github.com/src-d/go-mysql-server/sql/expression" 8 | "github.com/stretchr/testify/require" 9 | "gopkg.in/src-d/go-git.v4/plumbing" 10 | ) 11 | 12 | func TestReferencesTable(t *testing.T) { 13 | require := require.New(t) 14 | ctx, _, cleanup := setup(t) 15 | defer cleanup() 16 | 17 | table := newReferencesTable(poolFromCtx(t, ctx)) 18 | rows, err := tableToRows(ctx, table) 19 | require.NoError(err) 20 | 21 | for i := range rows { 22 | // remove repository id 23 | rows[i] = rows[i][1:] 24 | } 25 | 26 | expected := []sql.Row{ 27 | sql.NewRow("HEAD", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), 28 | sql.NewRow("refs/heads/master", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), 29 | sql.NewRow("refs/remotes/origin/branch", "e8d3ffab552895c19b9fcf7aa264d277cde33881"), 30 | sql.NewRow("refs/remotes/origin/master", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), 31 | } 32 | require.ElementsMatch(expected, rows) 33 | } 34 | 35 | func TestReferencesPushdown(t *testing.T) { 36 | require := require.New(t) 37 | ctx, _, cleanup := setup(t) 38 | defer cleanup() 39 | 40 | table := newReferencesTable(poolFromCtx(t, ctx)) 41 | 42 | rows, err := tableToRows(ctx, table) 43 | require.NoError(err) 44 | require.Len(rows, 4) 45 | 46 | t1 := table.WithFilters([]sql.Expression{ 47 | expression.NewEquals( 48 | expression.NewGetFieldWithTable(2, sql.Text, ReferencesTableName, "hash", false), 49 | expression.NewLiteral("e8d3ffab552895c19b9fcf7aa264d277cde33881", sql.Text), 50 | ), 51 | }) 52 | 53 | rows, err = tableToRows(ctx, t1) 54 | require.NoError(err) 55 | require.Len(rows, 1) 56 | 57 | t2 := table.WithFilters([]sql.Expression{ 58 | expression.NewEquals( 59 | expression.NewGetFieldWithTable(1, sql.Text, RepositoriesTableName, "name", false), 60 | expression.NewLiteral("refs/remotes/origin/master", sql.Text), 61 | ), 62 | }) 63 | 64 | rows, err = tableToRows(ctx, t2) 65 | require.NoError(err) 66 | require.Len(rows, 1) 67 | require.Equal("6ecf0ef2c2dffb796033e5a02219af86ec6584e5", rows[0][2]) 68 | 69 | t3 := table.WithFilters([]sql.Expression{ 70 | expression.NewEquals( 71 | expression.NewGetFieldWithTable(1, sql.Text, ReferencesTableName, "name", false), 72 | expression.NewLiteral("refs/remotes/origin/develop", sql.Text), 73 | ), 74 | }) 75 | 76 | rows, err = tableToRows(ctx, t3) 77 | require.NoError(err) 78 | require.Len(rows, 0) 79 | } 80 | 81 | func TestReferencesIndexKeyValueIter(t *testing.T) { 82 | require := require.New(t) 83 | ctx, _, cleanup := setup(t) 84 | defer cleanup() 85 | 86 | iter, err := newReferencesTable(poolFromCtx(t, ctx)). 87 | IndexKeyValues(ctx, []string{"ref_name"}) 88 | require.NoError(err) 89 | 90 | rows, err := tableToRows(ctx, newReferencesTable(poolFromCtx(t, ctx))) 91 | require.NoError(err) 92 | 93 | var expected []keyValue 94 | for _, row := range rows { 95 | var kv keyValue 96 | kv.key = assertEncodeRefsRow(t, row) 97 | kv.values = append(kv.values, row[1]) 98 | expected = append(expected, kv) 99 | } 100 | 101 | assertIndexKeyValueIter(t, iter, expected) 102 | } 103 | 104 | func assertEncodeRefsRow(t *testing.T, row sql.Row) []byte { 105 | t.Helper() 106 | k, err := new(refRowKeyMapper).fromRow(row) 107 | require.NoError(t, err) 108 | return k 109 | } 110 | 111 | func TestReferencesIndex(t *testing.T) { 112 | testTableIndex( 113 | t, 114 | new(referencesTable), 115 | []sql.Expression{expression.NewEquals( 116 | expression.NewGetField(1, sql.Text, "ref_name", false), 117 | expression.NewLiteral("HEAD", sql.Text), 118 | )}, 119 | ) 120 | } 121 | 122 | func TestRefRowKeyMapper(t *testing.T) { 123 | require := require.New(t) 124 | row := sql.Row{"repo1", "ref_name", plumbing.ZeroHash.String()} 125 | mapper := new(refRowKeyMapper) 126 | 127 | k, err := mapper.fromRow(row) 128 | require.NoError(err) 129 | 130 | row2, err := mapper.toRow(k) 131 | require.NoError(err) 132 | 133 | require.Equal(row, row2) 134 | } 135 | 136 | func TestReferencesIndexIterClosed(t *testing.T) { 137 | testTableIndexIterClosed(t, new(referencesTable)) 138 | } 139 | 140 | func TestReferencesIterClosed(t *testing.T) { 141 | testTableIterClosed(t, new(referencesTable)) 142 | } 143 | 144 | func TestReferencesIterators(t *testing.T) { 145 | // columns names just for debugging 146 | testTableIterators(t, new(referencesTable), []string{"ref_name", "commit_hash"}) 147 | } 148 | -------------------------------------------------------------------------------- /regression_test.go: -------------------------------------------------------------------------------- 1 | package gitbase_test 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | "github.com/src-d/gitbase" 10 | "github.com/src-d/go-mysql-server/sql" 11 | "github.com/src-d/go-mysql-server/sql/index/pilosa" 12 | "github.com/stretchr/testify/require" 13 | yaml "gopkg.in/yaml.v2" 14 | ) 15 | 16 | type Query struct { 17 | ID string `yaml:"ID"` 18 | Name string `yaml:"Name,omitempty"` 19 | Statements []string `yaml:"Statements"` 20 | } 21 | 22 | func TestRegressionQueries(t *testing.T) { 23 | _, pool, cleanup := setup(t) 24 | defer cleanup() 25 | 26 | engine := newSquashEngine(pool) 27 | tmpDir, err := ioutil.TempDir(os.TempDir(), "pilosa-idx-gitbase") 28 | require.NoError(t, err) 29 | defer os.RemoveAll(tmpDir) 30 | engine.Catalog.RegisterIndexDriver(pilosa.NewDriver(tmpDir)) 31 | 32 | ctx := sql.NewContext( 33 | context.TODO(), 34 | sql.WithSession(gitbase.NewSession(pool)), 35 | ) 36 | 37 | queries, err := loadQueriesYaml("./_testdata/regression.yml") 38 | require.NoError(t, err) 39 | 40 | for _, q := range queries { 41 | t.Run(q.ID, func(t *testing.T) { 42 | require := require.New(t) 43 | for _, stmt := range q.Statements { 44 | _, iter, err := engine.Query(ctx, stmt) 45 | if err != nil { 46 | require.Failf(err.Error(), "ID: %s, Name: %s, Statement: %s", q.ID, q.Name, stmt) 47 | } 48 | 49 | _, err = sql.RowIterToRows(iter) 50 | if err != nil { 51 | require.Failf(err.Error(), "ID: %s, Name: %s, Statement: %s", q.ID, q.Name, stmt) 52 | } 53 | } 54 | }) 55 | } 56 | } 57 | 58 | func loadQueriesYaml(file string) ([]Query, error) { 59 | text, err := ioutil.ReadFile(file) 60 | if err != nil { 61 | return nil, err 62 | } 63 | var q []Query 64 | err = yaml.Unmarshal(text, &q) 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | return q, nil 70 | } 71 | -------------------------------------------------------------------------------- /remotes_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/src-d/go-borges" 8 | "github.com/src-d/go-mysql-server/sql" 9 | "github.com/src-d/go-mysql-server/sql/expression" 10 | 11 | "github.com/stretchr/testify/require" 12 | gitconfig "gopkg.in/src-d/go-git.v4/config" 13 | ) 14 | 15 | func TestRemotesTable(t *testing.T) { 16 | require := require.New(t) 17 | ctx, fix, cleanup := setup(t) 18 | defer cleanup() 19 | 20 | table := newRemotesTable(poolFromCtx(t, ctx)) 21 | 22 | session := ctx.Session.(*Session) 23 | pool := session.Pool 24 | lib := pool.library 25 | 26 | bRepo, err := lib.Get(borges.RepositoryID(fix), borges.RWMode) 27 | require.NoError(err) 28 | r := bRepo.R() 29 | 30 | config := gitconfig.RemoteConfig{ 31 | Name: "my_remote", 32 | URLs: []string{"url1", "url2"}, 33 | Fetch: []gitconfig.RefSpec{ 34 | "refs/heads/*:refs/remotes/fetch1/*", 35 | "refs/heads/*:refs/remotes/fetch2/*", 36 | }, 37 | } 38 | 39 | _, err = r.CreateRemote(&config) 40 | require.NoError(err) 41 | 42 | err = bRepo.Close() 43 | require.NoError(err) 44 | 45 | repo, err := pool.GetRepo(fix) 46 | require.NoError(err) 47 | 48 | rows, err := tableToRows(ctx, table) 49 | require.NoError(err) 50 | require.Len(rows, 3) 51 | 52 | schema := table.Schema() 53 | for idx, row := range rows { 54 | err := schema.CheckRow(row) 55 | require.NoError(err, "row %d doesn't conform to schema", idx) 56 | 57 | if row[1] == "my_remote" { 58 | urlstring, ok := row[2].(string) 59 | require.True(ok) 60 | 61 | num := urlstring[len(urlstring)-1:] 62 | 63 | require.Equal(repo.ID(), row[0]) 64 | 65 | url := fmt.Sprintf("url%v", num) 66 | require.Equal(url, row[2]) // push 67 | require.Equal(url, row[3]) // fetch 68 | 69 | ref := fmt.Sprintf("refs/heads/*:refs/remotes/fetch%v/*", num) 70 | require.Equal(gitconfig.RefSpec(ref).String(), row[4]) // push 71 | require.Equal(gitconfig.RefSpec(ref).String(), row[5]) // fetch 72 | } else { 73 | require.Equal("origin", row[1]) 74 | } 75 | } 76 | } 77 | 78 | func TestRemotesPushdown(t *testing.T) { 79 | require := require.New(t) 80 | ctx, _, cleanup := setup(t) 81 | defer cleanup() 82 | 83 | table := newRemotesTable(poolFromCtx(t, ctx)) 84 | 85 | rows, err := tableToRows(ctx, table) 86 | require.NoError(err) 87 | require.Len(rows, 1) 88 | 89 | t1 := table.WithFilters([]sql.Expression{ 90 | expression.NewEquals( 91 | expression.NewGetField(1, sql.Text, "name", false), 92 | expression.NewLiteral("foo", sql.Text), 93 | ), 94 | }) 95 | 96 | rows, err = tableToRows(ctx, t1) 97 | require.NoError(err) 98 | require.Len(rows, 0) 99 | 100 | t2 := table.WithFilters([]sql.Expression{ 101 | expression.NewEquals( 102 | expression.NewGetField(1, sql.Text, "name", false), 103 | expression.NewLiteral("origin", sql.Text), 104 | ), 105 | }) 106 | 107 | rows, err = tableToRows(ctx, t2) 108 | require.NoError(err) 109 | require.Len(rows, 1) 110 | } 111 | 112 | func TestRemotesIndexKeyValueIter(t *testing.T) { 113 | require := require.New(t) 114 | ctx, path, cleanup := setup(t) 115 | defer cleanup() 116 | 117 | table := new(remotesTable) 118 | iter, err := table.IndexKeyValues(ctx, []string{"remote_name", "remote_push_url"}) 119 | require.NoError(err) 120 | 121 | var expected = []keyValue{ 122 | { 123 | key: assertEncodeKey(t, &remoteIndexKey{path, 0, 0}), 124 | values: []interface{}{"origin", "git@github.com:git-fixtures/basic.git"}, 125 | }, 126 | } 127 | 128 | assertIndexKeyValueIter(t, iter, expected) 129 | } 130 | 131 | func TestRemotesIndex(t *testing.T) { 132 | testTableIndex( 133 | t, 134 | new(remotesTable), 135 | []sql.Expression{expression.NewEquals( 136 | expression.NewGetField(1, sql.Text, "remote_name", false), 137 | expression.NewLiteral("foo", sql.Text), 138 | )}, 139 | ) 140 | } 141 | 142 | func TestEncodeRemoteIndexKey(t *testing.T) { 143 | require := require.New(t) 144 | 145 | k := remoteIndexKey{ 146 | Repository: "repo1", 147 | Pos: 5, 148 | URLPos: 7, 149 | } 150 | 151 | data, err := k.encode() 152 | require.NoError(err) 153 | 154 | var k2 remoteIndexKey 155 | require.NoError(k2.decode(data)) 156 | require.Equal(k, k2) 157 | } 158 | 159 | func TestRemotesIndexIterClosed(t *testing.T) { 160 | testTableIndexIterClosed(t, new(remotesTable)) 161 | } 162 | 163 | func TestRemotesIterators(t *testing.T) { 164 | // columns names just for debugging 165 | testTableIterators(t, new(remotesTable), []string{"remote_name", "remote_push_url"}) 166 | } 167 | -------------------------------------------------------------------------------- /repositories.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/src-d/go-mysql-server/sql" 7 | ) 8 | 9 | type repositoriesTable struct { 10 | checksumable 11 | partitioned 12 | filters []sql.Expression 13 | index sql.IndexLookup 14 | } 15 | 16 | // RepositoriesSchema is the schema for the repositories table. 17 | var RepositoriesSchema = sql.Schema{ 18 | {Name: "repository_id", Type: sql.Text, Nullable: false, Source: RepositoriesTableName}, 19 | } 20 | 21 | func newRepositoriesTable(pool *RepositoryPool) *repositoriesTable { 22 | return &repositoriesTable{checksumable: checksumable{pool}} 23 | } 24 | 25 | var _ Table = (*repositoriesTable)(nil) 26 | var _ Squashable = (*repositoriesTable)(nil) 27 | 28 | func (repositoriesTable) isSquashable() {} 29 | func (repositoriesTable) isGitbaseTable() {} 30 | 31 | func (repositoriesTable) Name() string { 32 | return RepositoriesTableName 33 | } 34 | 35 | func (repositoriesTable) Schema() sql.Schema { 36 | return RepositoriesSchema 37 | } 38 | 39 | func (r repositoriesTable) String() string { 40 | return printTable( 41 | RepositoriesTableName, 42 | RepositoriesSchema, 43 | nil, 44 | r.filters, 45 | r.index, 46 | ) 47 | } 48 | 49 | func (repositoriesTable) HandledFilters(filters []sql.Expression) []sql.Expression { 50 | return handledFilters(RepositoriesTableName, RepositoriesSchema, filters) 51 | } 52 | 53 | func (r *repositoriesTable) WithFilters(filters []sql.Expression) sql.Table { 54 | nt := *r 55 | nt.filters = filters 56 | return &nt 57 | } 58 | 59 | func (r *repositoriesTable) WithIndexLookup(idx sql.IndexLookup) sql.Table { 60 | nt := *r 61 | nt.index = idx 62 | return &nt 63 | } 64 | 65 | func (r *repositoriesTable) PartitionRows( 66 | ctx *sql.Context, 67 | p sql.Partition, 68 | ) (sql.RowIter, error) { 69 | repo, err := getPartitionRepo(ctx, p) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | span, ctx := ctx.Span("gitbase.RepositoriesTable") 75 | iter, err := rowIterWithSelectors( 76 | ctx, RepositoriesSchema, RepositoriesTableName, 77 | r.filters, 78 | r.handledColumns(), 79 | func(_ selectors) (sql.RowIter, error) { 80 | if r.index != nil { 81 | values, err := r.index.Values(p) 82 | if err != nil { 83 | return nil, err 84 | } 85 | return &rowIndexIter{new(repoRowKeyMapper), values}, nil 86 | } 87 | 88 | return &repositoriesRowIter{repo: repo}, nil 89 | }, 90 | ) 91 | 92 | if err != nil { 93 | span.Finish() 94 | return nil, errorWithRepo(repo, err) 95 | } 96 | 97 | return sql.NewSpanIter(span, newRepoRowIter(repo, iter)), nil 98 | } 99 | 100 | func (repositoriesTable) handledColumns() []string { return nil } 101 | 102 | func (r *repositoriesTable) IndexLookup() sql.IndexLookup { return r.index } 103 | func (r *repositoriesTable) Filters() []sql.Expression { return r.filters } 104 | 105 | // IndexKeyValues implements the sql.IndexableTable interface. 106 | func (r *repositoriesTable) IndexKeyValues( 107 | ctx *sql.Context, 108 | colNames []string, 109 | ) (sql.PartitionIndexKeyValueIter, error) { 110 | return newTablePartitionIndexKeyValueIter( 111 | ctx, 112 | newRepositoriesTable(r.pool), 113 | RepositoriesTableName, 114 | colNames, 115 | new(repoRowKeyMapper), 116 | ) 117 | } 118 | 119 | type repoRowKeyMapper struct{} 120 | 121 | func (repoRowKeyMapper) fromRow(row sql.Row) ([]byte, error) { 122 | if len(row) != 1 { 123 | return nil, errRowKeyMapperRowLength.New(1, len(row)) 124 | } 125 | 126 | repo, ok := row[0].(string) 127 | if !ok { 128 | return nil, errRowKeyMapperColType.New(0, repo, row[0]) 129 | } 130 | 131 | return []byte(repo), nil 132 | } 133 | 134 | func (repoRowKeyMapper) toRow(data []byte) (sql.Row, error) { 135 | return sql.Row{string(data)}, nil 136 | } 137 | 138 | type repositoriesRowIter struct { 139 | repo *Repository 140 | visited bool 141 | } 142 | 143 | func (i *repositoriesRowIter) Next() (sql.Row, error) { 144 | if i.visited { 145 | return nil, io.EOF 146 | } 147 | 148 | i.visited = true 149 | return sql.NewRow(i.repo.ID()), nil 150 | } 151 | 152 | func (i *repositoriesRowIter) Close() error { 153 | i.visited = true 154 | if i.repo != nil { 155 | i.repo.Close() 156 | } 157 | return nil 158 | } 159 | -------------------------------------------------------------------------------- /repositories_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | fixtures "github.com/src-d/go-git-fixtures" 8 | "github.com/src-d/go-mysql-server/sql" 9 | "github.com/src-d/go-mysql-server/sql/expression" 10 | "github.com/stretchr/testify/require" 11 | "gopkg.in/src-d/go-git.v4/plumbing/cache" 12 | ) 13 | 14 | func TestRepositoriesTable(t *testing.T) { 15 | require := require.New(t) 16 | 17 | repoIDs := []string{ 18 | "one", "two", "three", "four", "five", "six", 19 | "seven", "eight", "nine", 20 | } 21 | 22 | lib, err := newMultiLibrary() 23 | require.NoError(err) 24 | 25 | pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib) 26 | 27 | path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() 28 | for _, id := range repoIDs { 29 | err = lib.AddPlain(id, path, nil) 30 | require.NoError(err) 31 | } 32 | 33 | session := NewSession(pool) 34 | ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) 35 | 36 | db := NewDatabase(RepositoriesTableName, poolFromCtx(t, ctx)) 37 | require.NotNil(db) 38 | 39 | tables := db.Tables() 40 | table, ok := tables[RepositoriesTableName] 41 | 42 | require.True(ok) 43 | require.NotNil(table) 44 | 45 | rows, err := tableToRows(ctx, table) 46 | require.NoError(err) 47 | require.Len(rows, len(repoIDs)) 48 | 49 | idArray := make([]string, len(repoIDs)) 50 | for i, row := range rows { 51 | idArray[i] = row[0].(string) 52 | } 53 | require.ElementsMatch(idArray, repoIDs) 54 | 55 | schema := table.Schema() 56 | for idx, row := range rows { 57 | err := schema.CheckRow(row) 58 | require.NoError(err, "row %d doesn't conform to schema", idx) 59 | } 60 | } 61 | 62 | func TestRepositoriesPushdown(t *testing.T) { 63 | require := require.New(t) 64 | ctx, path, cleanup := setup(t) 65 | defer cleanup() 66 | 67 | table := newRepositoriesTable(poolFromCtx(t, ctx)) 68 | 69 | rows, err := tableToRows(ctx, table) 70 | require.NoError(err) 71 | require.Len(rows, 1) 72 | 73 | t1 := table.WithFilters([]sql.Expression{ 74 | expression.NewEquals( 75 | expression.NewGetField(0, sql.Text, "id", false), 76 | expression.NewLiteral("foo", sql.Text), 77 | ), 78 | }) 79 | 80 | rows, err = tableToRows(ctx, t1) 81 | require.NoError(err) 82 | require.Len(rows, 0) 83 | 84 | t2 := table.WithFilters([]sql.Expression{ 85 | expression.NewEquals( 86 | expression.NewGetField(0, sql.Text, "id", false), 87 | expression.NewLiteral(path, sql.Text), 88 | ), 89 | }) 90 | 91 | rows, err = tableToRows(ctx, t2) 92 | require.NoError(err) 93 | require.Len(rows, 1) 94 | } 95 | 96 | func TestRepositoriesIndexKeyValueIter(t *testing.T) { 97 | require := require.New(t) 98 | ctx, path, cleanup := setup(t) 99 | defer cleanup() 100 | 101 | iter, err := new(repositoriesTable).IndexKeyValues(ctx, []string{"repository_id"}) 102 | require.NoError(err) 103 | 104 | assertIndexKeyValueIter(t, iter, 105 | []keyValue{ 106 | { 107 | assertEncodeRepoRow(t, sql.NewRow(path)), 108 | []interface{}{path}, 109 | }, 110 | }, 111 | ) 112 | } 113 | 114 | func assertEncodeRepoRow(t *testing.T, row sql.Row) []byte { 115 | t.Helper() 116 | k, err := new(repoRowKeyMapper).fromRow(row) 117 | require.NoError(t, err) 118 | return k 119 | } 120 | 121 | func TestRepositoriesIndex(t *testing.T) { 122 | testTableIndex( 123 | t, 124 | new(repositoriesTable), 125 | []sql.Expression{ 126 | expression.NewEquals( 127 | expression.NewGetFieldWithTable(0, sql.Text, RepositoriesTableName, "repository_id", false), 128 | expression.NewLiteral("non-existent-repo", sql.Text), 129 | ), 130 | }, 131 | ) 132 | } 133 | -------------------------------------------------------------------------------- /repository_pool.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/src-d/go-borges" 8 | billy "gopkg.in/src-d/go-billy.v4" 9 | errors "gopkg.in/src-d/go-errors.v1" 10 | git "gopkg.in/src-d/go-git.v4" 11 | "gopkg.in/src-d/go-git.v4/plumbing/cache" 12 | "gopkg.in/src-d/go-git.v4/storage/filesystem" 13 | ) 14 | 15 | var ( 16 | errRepoAlreadyRegistered = errors.NewKind("the repository is already registered: %s") 17 | 18 | gitStorerOptions = filesystem.Options{ 19 | ExclusiveAccess: true, 20 | KeepDescriptors: true, 21 | } 22 | ) 23 | 24 | type Repository struct { 25 | *git.Repository 26 | 27 | cache cache.Object 28 | repo borges.Repository 29 | lib borges.Library 30 | } 31 | 32 | func NewRepository( 33 | lib borges.Library, 34 | repo borges.Repository, 35 | cache cache.Object, 36 | ) *Repository { 37 | return &Repository{ 38 | Repository: repo.R(), 39 | lib: lib, 40 | repo: repo, 41 | cache: cache, 42 | } 43 | } 44 | 45 | func (r *Repository) ID() string { 46 | return r.repo.ID().String() 47 | } 48 | 49 | func (r *Repository) FS() (billy.Filesystem, error) { 50 | fs := r.repo.FS() 51 | if fs == nil { 52 | return nil, fmt.Errorf("filesystem inaccesible") 53 | } 54 | 55 | return fs, nil 56 | } 57 | 58 | func (r *Repository) Cache() cache.Object { 59 | return r.cache 60 | } 61 | 62 | func (r *Repository) Close() error { 63 | if r != nil && r.repo != nil { 64 | if closer, ok := r.repo.(io.Closer); ok { 65 | return closer.Close() 66 | } 67 | } 68 | return nil 69 | } 70 | 71 | // RepositoryPool holds a pool git repository paths and 72 | // functionality to open and iterate them. 73 | type RepositoryPool struct { 74 | cache cache.Object 75 | library borges.Library 76 | } 77 | 78 | // NewRepositoryPool holds a repository library and a shared object cache. 79 | func NewRepositoryPool( 80 | c cache.Object, 81 | lib borges.Library, 82 | ) *RepositoryPool { 83 | return &RepositoryPool{ 84 | cache: c, 85 | library: lib, 86 | } 87 | } 88 | 89 | // ErrPoolRepoNotFound is returned when a repository id is not present in the pool. 90 | var ErrPoolRepoNotFound = errors.NewKind("repository id %s not found in the pool") 91 | 92 | // GetRepo returns a repository with the given id from the pool. 93 | func (p *RepositoryPool) GetRepo(id string) (*Repository, error) { 94 | i := borges.RepositoryID(id) 95 | 96 | repo, err := p.library.Get(i, borges.ReadOnlyMode) 97 | if err != nil { 98 | if borges.ErrRepositoryNotExists.Is(err) { 99 | return nil, ErrPoolRepoNotFound.New(id) 100 | } 101 | 102 | return nil, err 103 | } 104 | 105 | r := NewRepository(p.library, repo, p.cache) 106 | return r, nil 107 | } 108 | 109 | // RepoIter creates a new Repository iterator 110 | func (p *RepositoryPool) RepoIter() (*RepositoryIter, error) { 111 | it, err := p.library.Repositories(borges.ReadOnlyMode) 112 | if err != nil { 113 | return nil, err 114 | } 115 | 116 | iter := &RepositoryIter{ 117 | pool: p, 118 | iter: it, 119 | } 120 | 121 | return iter, nil 122 | } 123 | 124 | // RepositoryIter iterates over all repositories in the pool 125 | type RepositoryIter struct { 126 | pool *RepositoryPool 127 | iter borges.RepositoryIterator 128 | } 129 | 130 | // Next retrieves the next Repository. It returns io.EOF as error 131 | // when there are no more Repositories to retrieve. 132 | func (i *RepositoryIter) Next() (*Repository, error) { 133 | repo, err := i.iter.Next() 134 | if err != nil { 135 | return nil, err 136 | } 137 | 138 | r := NewRepository(i.pool.library, repo, i.pool.cache) 139 | return r, nil 140 | } 141 | 142 | // Close finished iterator. It's no-op. 143 | func (i *RepositoryIter) Close() error { 144 | return nil 145 | } 146 | -------------------------------------------------------------------------------- /repository_pool_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "testing" 9 | 10 | fixtures "github.com/src-d/go-git-fixtures" 11 | "github.com/stretchr/testify/require" 12 | "gopkg.in/src-d/go-git.v4/plumbing/cache" 13 | "gopkg.in/src-d/go-git.v4/plumbing/object" 14 | ) 15 | 16 | func TestRepositoryPoolBasic(t *testing.T) { 17 | require := require.New(t) 18 | 19 | lib, err := newMultiLibrary() 20 | require.NoError(err) 21 | 22 | pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib) 23 | 24 | iter, err := pool.RepoIter() 25 | require.NoError(err) 26 | 27 | repo, err := iter.Next() 28 | require.Nil(repo) 29 | require.Equal(io.EOF, err) 30 | 31 | repo, err = pool.GetRepo("foo") 32 | require.Nil(repo) 33 | require.EqualError(err, ErrPoolRepoNotFound.New("foo").Error()) 34 | 35 | repo, err = pool.GetRepo("directory/should/not/exist") 36 | require.Nil(repo) 37 | require.EqualError(err, ErrPoolRepoNotFound.New("directory/should/not/exist").Error()) 38 | 39 | path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() 40 | 41 | err = lib.AddPlain("1", path, nil) 42 | require.NoError(err) 43 | 44 | iter, err = pool.RepoIter() 45 | require.NoError(err) 46 | repo, err = iter.Next() 47 | require.NoError(err) 48 | require.Equal("1", repo.ID()) 49 | require.NotNil(repo) 50 | 51 | repo, err = pool.GetRepo("1") 52 | require.NoError(err) 53 | require.Equal("1", repo.ID()) 54 | require.NotNil(repo) 55 | 56 | _, err = iter.Next() 57 | require.Equal(io.EOF, err) 58 | } 59 | 60 | func TestRepositoryPoolGit(t *testing.T) { 61 | require := require.New(t) 62 | 63 | path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() 64 | 65 | lib, err := newMultiLibrary() 66 | require.NoError(err) 67 | 68 | pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib) 69 | 70 | require.NoError(lib.AddPlain(path, path, nil)) 71 | 72 | riter, err := pool.RepoIter() 73 | require.NoError(err) 74 | repo, err := riter.Next() 75 | name := strings.TrimLeft(path, string(os.PathSeparator)) 76 | require.Equal(name, repo.ID()) 77 | require.NotNil(repo) 78 | require.NoError(err) 79 | 80 | iter, err := newCommitIter(repo, false) 81 | require.NoError(err) 82 | 83 | count := 0 84 | 85 | for { 86 | commit, err := iter.Next() 87 | if err != nil { 88 | break 89 | } 90 | 91 | require.NotNil(commit) 92 | 93 | count++ 94 | } 95 | 96 | require.Equal(9, count) 97 | } 98 | 99 | func TestRepositoryPoolIterator(t *testing.T) { 100 | require := require.New(t) 101 | 102 | path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() 103 | 104 | lib, err := newMultiLibrary() 105 | require.NoError(err) 106 | 107 | pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib) 108 | err = lib.AddPlain("0", path, nil) 109 | require.NoError(err) 110 | err = lib.AddPlain("1", path, nil) 111 | require.NoError(err) 112 | 113 | iter, err := pool.RepoIter() 114 | require.NoError(err) 115 | 116 | count := 0 117 | 118 | var ids []string 119 | for { 120 | repo, err := iter.Next() 121 | if err != nil { 122 | require.Equal(io.EOF, err) 123 | break 124 | } 125 | 126 | require.NotNil(repo) 127 | ids = append(ids, repo.ID()) 128 | 129 | count++ 130 | } 131 | 132 | require.Equal(2, count) 133 | require.ElementsMatch([]string{"0", "1"}, ids) 134 | } 135 | 136 | func TestRepositoryPoolSiva(t *testing.T) { 137 | require := require.New(t) 138 | 139 | lib, err := newMultiLibrary() 140 | require.NoError(err) 141 | 142 | cwd, err := os.Getwd() 143 | require.NoError(err) 144 | 145 | pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib) 146 | path := filepath.Join(cwd, "_testdata") 147 | 148 | require.NoError( 149 | filepath.Walk(path, func(path string, info os.FileInfo, err error) error { 150 | if err != nil { 151 | return err 152 | } 153 | 154 | if IsSivaFile(path) { 155 | require.NoError(lib.AddSiva(path, nil)) 156 | } 157 | 158 | return nil 159 | }), 160 | ) 161 | 162 | expectedRepos := 5 163 | expected := map[string]int{ 164 | "015da2f4-6d89-7ec8-5ac9-a38329ea875b": 606, 165 | "015dcc49-9049-b00c-ba72-b6f5fa98cbe7": 71, 166 | "015dcc49-90e6-34f2-ac03-df879ee269f3": 45, 167 | "015dcc4d-0bdf-6aff-4aac-ffe68c752eb3": 382, 168 | "015dcc4d-2622-bdac-12a5-ec441e3f3508": 72, 169 | } 170 | result := make(map[string]int) 171 | 172 | it, err := pool.RepoIter() 173 | require.NoError(err) 174 | 175 | var i int 176 | for { 177 | repo, err := it.Next() 178 | if err == io.EOF { 179 | break 180 | } 181 | require.NoError(err) 182 | 183 | iter, err := newCommitIter(repo, false) 184 | require.NoError(err) 185 | 186 | id := repo.ID() 187 | result[id] = 0 188 | require.NoError(iter.ForEach(func(c *object.Commit) error { 189 | result[id]++ 190 | return nil 191 | })) 192 | 193 | i++ 194 | } 195 | 196 | require.Equal(expectedRepos, i) 197 | require.Equal(expected, result) 198 | } 199 | -------------------------------------------------------------------------------- /session_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | "google.golang.org/grpc/connectivity" 9 | ) 10 | 11 | func TestSessionBblfshClient(t *testing.T) { 12 | require := require.New(t) 13 | 14 | session := NewSession(nil, WithBblfshEndpoint(defaultBblfshEndpoint)) 15 | cli, err := session.BblfshClient() 16 | require.NoError(err) 17 | require.NotNil(cli) 18 | require.Equal(connectivity.Ready, cli.GetState()) 19 | } 20 | 21 | func TestSupportedLanguagesAliases(t *testing.T) { 22 | require := require.New(t) 23 | 24 | session := NewSession(nil, WithBblfshEndpoint(defaultBblfshEndpoint)) 25 | cli, err := session.BblfshClient() 26 | require.NoError(err) 27 | require.NotNil(cli) 28 | require.Equal(connectivity.Ready, cli.GetState()) 29 | ok, err := cli.IsLanguageSupported(context.TODO(), "C++") 30 | require.NoError(err) 31 | require.True(ok) 32 | } 33 | 34 | func TestSessionBblfshClientNoConnection(t *testing.T) { 35 | require := require.New(t) 36 | 37 | session := NewSession(nil, WithBblfshEndpoint("localhost:9999")) 38 | _, err := session.BblfshClient() 39 | require.Error(err) 40 | require.True(ErrBblfshConnection.Is(err)) 41 | } 42 | -------------------------------------------------------------------------------- /squash.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/src-d/go-mysql-server/sql" 8 | ) 9 | 10 | // SquashedTable is a table that combines the output of some tables as the 11 | // inputs of others with chaining so it's less expensive to compute. 12 | type SquashedTable struct { 13 | partitioned 14 | iter ChainableIter 15 | tables []string 16 | schemaMappings []int 17 | filters []sql.Expression 18 | indexedTables []string 19 | schema sql.Schema 20 | } 21 | 22 | // NewSquashedTable creates a new SquashedTable. 23 | func NewSquashedTable( 24 | iter ChainableIter, 25 | mapping []int, 26 | filters []sql.Expression, 27 | indexedTables []string, 28 | tables ...string, 29 | ) *SquashedTable { 30 | return &SquashedTable{ 31 | iter: iter, 32 | tables: tables, 33 | schemaMappings: mapping, 34 | filters: filters, 35 | indexedTables: indexedTables, 36 | } 37 | } 38 | 39 | var _ sql.Table = (*SquashedTable)(nil) 40 | var _ sql.PartitionCounter = (*SquashedTable)(nil) 41 | 42 | // Name implements the sql.Table interface. 43 | func (t *SquashedTable) Name() string { 44 | return fmt.Sprintf("SquashedTable(%s)", strings.Join(t.tables, ", ")) 45 | } 46 | 47 | // Schema implements the sql.Table interface. 48 | func (t *SquashedTable) Schema() sql.Schema { 49 | if len(t.schemaMappings) == 0 { 50 | return t.iter.Schema() 51 | } 52 | 53 | if t.schema == nil { 54 | schema := t.iter.Schema() 55 | t.schema = make(sql.Schema, len(schema)) 56 | for i, j := range t.schemaMappings { 57 | t.schema[i] = schema[j] 58 | } 59 | } 60 | 61 | return t.schema 62 | } 63 | 64 | // PartitionRows implements the sql.Table interface. 65 | func (t *SquashedTable) PartitionRows(ctx *sql.Context, p sql.Partition) (sql.RowIter, error) { 66 | span, ctx := ctx.Span("gitbase.SquashedTable") 67 | 68 | session, err := getSession(ctx) 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | repo, err := getPartitionRepo(ctx, p) 74 | if err != nil { 75 | span.Finish() 76 | if session.SkipGitErrors { 77 | return noRows, nil 78 | } 79 | 80 | return nil, err 81 | } 82 | 83 | iter, err := t.iter.New(ctx, repo) 84 | if err != nil { 85 | span.Finish() 86 | return nil, errorWithRepo(repo, err) 87 | } 88 | 89 | if len(t.schemaMappings) == 0 { 90 | return sql.NewSpanIter( 91 | span, 92 | newRepoRowIter(repo, NewChainableRowIter(iter)), 93 | ), nil 94 | } 95 | 96 | return sql.NewSpanIter( 97 | span, 98 | newRepoRowIter( 99 | repo, 100 | NewSchemaMapperIter(NewChainableRowIter(iter), t.schemaMappings), 101 | ), 102 | ), nil 103 | } 104 | 105 | func (t *SquashedTable) String() string { 106 | s := t.Schema() 107 | cp := sql.NewTreePrinter() 108 | _ = cp.WriteNode("Columns") 109 | var schema = make([]string, len(s)) 110 | for i, col := range s { 111 | schema[i] = fmt.Sprintf( 112 | "Column(%s, %s, nullable=%v)", 113 | col.Name, 114 | col.Type.Type().String(), 115 | col.Nullable, 116 | ) 117 | } 118 | _ = cp.WriteChildren(schema...) 119 | 120 | fp := sql.NewTreePrinter() 121 | _ = fp.WriteNode("Filters") 122 | var filters = make([]string, len(t.filters)) 123 | for i, f := range t.filters { 124 | filters[i] = f.String() 125 | } 126 | _ = fp.WriteChildren(filters...) 127 | 128 | children := []string{cp.String(), fp.String()} 129 | 130 | if len(t.indexedTables) > 0 { 131 | ip := sql.NewTreePrinter() 132 | _ = ip.WriteNode("IndexedTables") 133 | _ = ip.WriteChildren(t.indexedTables...) 134 | children = append(children, ip.String()) 135 | } 136 | 137 | p := sql.NewTreePrinter() 138 | _ = p.WriteNode("SquashedTable(%s)", strings.Join(t.tables, ", ")) 139 | _ = p.WriteChildren(children...) 140 | return p.String() 141 | } 142 | 143 | type chainableRowIter struct { 144 | ChainableIter 145 | } 146 | 147 | // NewChainableRowIter converts a ChainableIter into a sql.RowIter. 148 | func NewChainableRowIter(iter ChainableIter) sql.RowIter { 149 | return &chainableRowIter{iter} 150 | } 151 | 152 | func (i *chainableRowIter) Next() (sql.Row, error) { 153 | if err := i.Advance(); err != nil { 154 | return nil, err 155 | } 156 | 157 | return i.Row(), nil 158 | } 159 | 160 | type schemaMapperIter struct { 161 | iter sql.RowIter 162 | mappings []int 163 | } 164 | 165 | // NewSchemaMapperIter reorders the rows in the given row iter according to the 166 | // given column mappings. 167 | func NewSchemaMapperIter(iter sql.RowIter, mappings []int) sql.RowIter { 168 | return &schemaMapperIter{iter, mappings} 169 | } 170 | 171 | func (i schemaMapperIter) Next() (sql.Row, error) { 172 | childRow, err := i.iter.Next() 173 | if err != nil { 174 | return nil, err 175 | } 176 | 177 | if len(i.mappings) == 0 { 178 | return childRow, nil 179 | } 180 | 181 | var row = make(sql.Row, len(i.mappings)) 182 | for i, j := range i.mappings { 183 | row[i] = childRow[j] 184 | } 185 | 186 | return row, nil 187 | } 188 | func (i schemaMapperIter) Close() error { return i.iter.Close() } 189 | -------------------------------------------------------------------------------- /table.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/src-d/go-mysql-server/sql" 7 | ) 8 | 9 | // Table represents a gitbase table. 10 | type Table interface { 11 | sql.FilteredTable 12 | sql.Checksumable 13 | sql.PartitionCounter 14 | gitBase 15 | } 16 | 17 | // Squashable represents a table that can be squashed. 18 | type Squashable interface { 19 | isSquashable() 20 | } 21 | 22 | type gitBase interface { 23 | isGitbaseTable() 24 | } 25 | 26 | func printTable( 27 | name string, 28 | tableSchema sql.Schema, 29 | projection []string, 30 | filters []sql.Expression, 31 | index sql.IndexLookup, 32 | ) string { 33 | p := sql.NewTreePrinter() 34 | _ = p.WriteNode("Table(%s)", name) 35 | var children = make([]string, len(tableSchema)) 36 | for i, col := range tableSchema { 37 | children[i] = fmt.Sprintf( 38 | "Column(%s, %s, nullable=%v)", 39 | col.Name, 40 | col.Type.Type().String(), 41 | col.Nullable, 42 | ) 43 | } 44 | 45 | if len(projection) > 0 { 46 | children = append(children, printableProjection(projection)) 47 | } 48 | 49 | if len(filters) > 0 { 50 | children = append(children, printableFilters(filters)) 51 | } 52 | 53 | if index != nil { 54 | children = append(children, printableIndexes(index)) 55 | } 56 | 57 | _ = p.WriteChildren(children...) 58 | return p.String() 59 | } 60 | 61 | func printableFilters(filters []sql.Expression) string { 62 | p := sql.NewTreePrinter() 63 | _ = p.WriteNode("Filters") 64 | var fs = make([]string, len(filters)) 65 | for i, f := range filters { 66 | fs[i] = f.String() 67 | } 68 | _ = p.WriteChildren(fs...) 69 | return p.String() 70 | } 71 | 72 | func printableProjection(projection []string) string { 73 | p := sql.NewTreePrinter() 74 | _ = p.WriteNode("Projected") 75 | _ = p.WriteChildren(projection...) 76 | return p.String() 77 | } 78 | 79 | func printableIndexes(idx sql.IndexLookup) string { 80 | p := sql.NewTreePrinter() 81 | _ = p.WriteNode("Indexes") 82 | _ = p.WriteChildren(idx.Indexes()...) 83 | return p.String() 84 | } 85 | -------------------------------------------------------------------------------- /table_test.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/src-d/go-mysql-server/sql" 8 | ) 9 | 10 | const expectedString = `Table(foo) 11 | ├─ Column(col1, TEXT, nullable=true) 12 | └─ Column(col2, INT64, nullable=false) 13 | ` 14 | 15 | func TestTableString(t *testing.T) { 16 | require := require.New(t) 17 | schema := sql.Schema{ 18 | {Name: "col1", Type: sql.Text, Nullable: true}, 19 | {Name: "col2", Type: sql.Int64}, 20 | } 21 | require.Equal(expectedString, printTable("foo", schema, nil, nil, nil)) 22 | } 23 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package gitbase 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/src-d/go-mysql-server/sql" 7 | ) 8 | 9 | var noRows emptyRowIter 10 | 11 | type emptyRowIter struct{} 12 | 13 | func (emptyRowIter) Next() (sql.Row, error) { return nil, io.EOF } 14 | func (emptyRowIter) Close() error { return nil } 15 | --------------------------------------------------------------------------------