├── .editorconfig ├── .env.example ├── .github ├── dependabot.yml └── workflows │ ├── golang.yml │ └── greeting.yml ├── .gitignore ├── .goreleaser.yml ├── .vscode ├── extensions.json ├── launch.json └── settings.json ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── cmd.go ├── cmd_bitbucket.go ├── cmd_github.go ├── cmd_gitlab.go └── tabia │ └── main.go ├── go.mod ├── go.sum ├── lib ├── bitbucket │ ├── bitbucket.go │ ├── bitbucket_test.go │ ├── projects.go │ ├── projects_test.go │ ├── repositories.go │ └── repositories_test.go ├── github │ ├── client.go │ ├── client_test.go │ ├── contents.go │ ├── contents_test.go │ ├── filter.go │ ├── filter_test.go │ ├── graphql │ │ ├── graphql.go │ │ ├── members.go │ │ └── search.go │ ├── members.go │ ├── members_test.go │ ├── repositories.go │ └── repositories_test.go ├── gitlab │ ├── client.go │ ├── filter.go │ ├── filter_test.go │ ├── repositories.go │ └── repositories_test.go ├── grimoirelab │ ├── bitbucket.go │ ├── bitbucket_test.go │ ├── github.go │ ├── github_test.go │ ├── grimoirelab.go │ └── grimoirelab_test.go ├── output │ ├── output.go │ └── output_test.go ├── shared │ ├── visibility.go │ ├── visibility_string.go │ └── visibility_test.go └── transport │ ├── transport.go │ └── transport_test.go └── tools.go /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | trim_trailing_whitespace = true 7 | insert_final_newline = true 8 | 9 | [*.go] 10 | indent_style = tab 11 | indent_size = 4 12 | tab_width = 4 13 | 14 | [Makefile] 15 | indent_style = tab 16 | indent_size = 8 17 | tab_width = 8 18 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | export TABIA_BITBUCKET_API=https://bitbucket.your.instance.com/rest/api/1.0 2 | export TABIA_BITBUCKET_USER= 3 | export TABIA_BITBUCKET_TOKEN= 4 | export TABIA_GITHUB_USER= 5 | export TABIA_GITHUB_TOKEN= 6 | export TABIA_GITLAB_INSTANCE=https://gitlab.your.instance.com/ 7 | export TABIA_GITLAB_TOKEN= 8 | # when requests should go via proxy 9 | # export SOCKS_PROXY=localhost:1080 10 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | 13 | - package-ecosystem: "github-actions" 14 | # Workflow files stored in the 15 | # default location of `.github/workflows` 16 | directory: "/" 17 | schedule: 18 | interval: "weekly" 19 | -------------------------------------------------------------------------------- /.github/workflows/golang.yml: -------------------------------------------------------------------------------- 1 | name: Go CI 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | 11 | name: Continuous Integration 12 | if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository 13 | 14 | permissions: 15 | contents: read 16 | 17 | steps: 18 | - name: Check out code 19 | uses: actions/checkout@v4 20 | 21 | - name: Set up Go 22 | uses: actions/setup-go@v5.5.0 23 | with: 24 | go-version-file: go.mod 25 | check-latest: true 26 | 27 | - name: Lint 28 | run: | 29 | go install golang.org/x/tools/cmd/goimports 30 | result=$($(go env GOPATH)/bin/goimports -d -e -local github.com/philips-labs/tabia $(go list -f {{.Dir}} ./...)) 31 | echo $result 32 | [ -n "$result" ] && exit 1 || exit 0 33 | 34 | - name: Get dependencies 35 | run: go mod download 36 | 37 | - name: Install tools 38 | run: make install-tools 39 | 40 | - name: Build 41 | run: | 42 | make build 43 | 44 | - name: Test and Cover 45 | run: go test -v -race -count=1 -covermode=atomic -coverprofile=coverage.out ./... 46 | env: 47 | TABIA_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 48 | 49 | - name: Upload Code Coverage 50 | uses: codecov/codecov-action@v5.4.3 51 | with: 52 | token: ${{ secrets.CODECOV_TOKEN }} 53 | files: ./coverage.out 54 | flags: unittests 55 | name: codecov-umbrella 56 | fail_ci_if_error: true 57 | verbose: true 58 | 59 | release: 60 | name: release 61 | needs: [build] 62 | runs-on: ubuntu-latest 63 | if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository 64 | 65 | permissions: 66 | contents: write 67 | 68 | steps: 69 | - name: Checkout 70 | uses: actions/checkout@v4 71 | with: 72 | fetch-depth: 0 73 | 74 | - name: Set up Go 75 | uses: actions/setup-go@v5.5.0 76 | with: 77 | go-version-file: go.mod 78 | check-latest: true 79 | 80 | - name: Login to GitHub Container Registry 81 | uses: docker/login-action@v3 82 | with: 83 | registry: ghcr.io 84 | username: ${{ github.actor }} 85 | password: ${{ secrets.GITHUB_TOKEN }} 86 | 87 | - name: Release 88 | uses: goreleaser/goreleaser-action@v5 89 | with: 90 | version: latest 91 | args: release --clean 92 | env: 93 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 94 | 95 | - name: Logout from DockerHub Registry 96 | if: ${{ always() }} 97 | run: docker logout 98 | -------------------------------------------------------------------------------- /.github/workflows/greeting.yml: -------------------------------------------------------------------------------- 1 | name: Greetings 2 | 3 | on: [issues, pull_request] 4 | 5 | jobs: 6 | greeting: 7 | runs-on: ubuntu-20.04 8 | steps: 9 | - uses: actions/first-interaction@v1.3.0 10 | with: 11 | repo-token: ${{ secrets.GITHUB_TOKEN }} 12 | issue-message: 'Thank you for submitting your first issue. We will be looking into it as soon as possible.' 13 | pr-message: 'Thanks for your first PR. We really appreciate it!' 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | dist/ 3 | .env 4 | *.out 5 | github-project-matching.json 6 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # This is an example goreleaser.yaml file with some sane defaults. 2 | # Make sure to check the documentation at http://goreleaser.com 3 | project_name: tabia 4 | 5 | before: 6 | hooks: 7 | - go mod download 8 | 9 | env: 10 | - CGO_ENABLED=0 11 | 12 | builds: 13 | - id: binary 14 | dir: cmd/tabia 15 | goos: 16 | - windows 17 | - darwin 18 | - linux 19 | goarch: 20 | - amd64 21 | - arm64 22 | goarm: 23 | - 8 24 | ldflags: 25 | - -s -w -X main.version={{.Version}} 26 | 27 | archives: 28 | - id: archive 29 | format: tar.gz 30 | files: 31 | - LICENSE* 32 | - README* 33 | format_overrides: 34 | - goos: windows 35 | format: zip 36 | 37 | dockers: 38 | - goos: linux 39 | goarch: amd64 40 | image_templates: 41 | - "ghcr.io/philips-labs/{{ .ProjectName }}:{{ .Tag }}" 42 | - "ghcr.io/philips-labs/{{ .ProjectName }}:v{{ .Major }}" 43 | - "ghcr.io/philips-labs/{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}" 44 | - "ghcr.io/philips-labs/{{ .ProjectName }}:latest" 45 | build_flag_templates: 46 | - "--pull" 47 | - "--label=com.opencontainers.image.created={{.Date}}" 48 | - "--label=org.opencontainers.image.title={{.ProjectName}}" 49 | - "--label=org.opencontainers.image.revision={{.FullCommit}}" 50 | - "--label=org.opencontainers.image.version={{.Version}}" 51 | - "--build-arg=VERSION={{.Version}}" 52 | extra_files: 53 | - "go.mod" 54 | - "go.sum" 55 | - "cmd" 56 | - "lib" 57 | checksum: 58 | name_template: 'checksums.txt' 59 | 60 | snapshot: 61 | name_template: "{{ .Tag }}-next" 62 | 63 | changelog: 64 | sort: asc 65 | filters: 66 | exclude: 67 | - '^docs:' 68 | - '^test:' 69 | - Merge pull request 70 | - Merge branch 71 | 72 | release: 73 | prerelease: auto 74 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | // See http://go.microsoft.com/fwlink/?LinkId=827846 3 | // for the documentation about the extensions.json format 4 | "recommendations": [ 5 | // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp 6 | "golang.go", 7 | "editorconfig.editorconfig", 8 | "davidanson.vscode-markdownlint" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch", 9 | "type": "go", 10 | "request": "launch", 11 | "mode": "auto", 12 | "program": "${fileDirname}", 13 | "env": {}, 14 | "args": [] 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "go.formatTool": "goimports", 3 | "go.testEnvFile": "${workspaceFolder}/.env" 4 | } 5 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.18-alpine as builder 2 | RUN mkdir build 3 | WORKDIR /build 4 | COPY go.mod go.sum ./ 5 | RUN go mod download 6 | ARG VERSION=dev-docker 7 | COPY . . 8 | RUN CGO_ENABLED=0 go build -v -trimpath -ldflags "-X 'main.version=${VERSION}'" -o bin/tabia ./cmd/tabia 9 | 10 | FROM alpine 11 | LABEL maintainer="marco.franssen@philips.com" 12 | RUN mkdir -p /app/data 13 | WORKDIR /app 14 | VOLUME [ "/app/data" ] 15 | ENV TABIA_BITBUCKET_API=\ 16 | TABIA_BITBUCKET_USER=\ 17 | TABIA_BITBUCKET_TOKEN=\ 18 | TABIA_GITHUB_USER=\ 19 | TABIA_GITHUB_TOKEN= 20 | COPY --from=builder build/bin/tabia . 21 | ENTRYPOINT [ "/app/tabia" ] 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Koninklijke Philips N.V, https://www.philips.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help clean build test test-cover fmt coverage-out coverage-html 2 | 3 | export GO111MODULE=on 4 | export CGO_ENABLED=0 5 | 6 | SRCS = $(shell git ls-files '*.go' | grep -v '^vendor/') 7 | 8 | MAIN_DIRECTORY:=./cmd/tabia 9 | BIN_OUTPUT:=$(if $(filter $(shell go env GOOS), Windows), bin/tabia.exe, bin/tabia) 10 | 11 | TAG_NAME:=$(shell git tag -l --contains HEAD) 12 | SHA:=$(shell git rev-parse HEAD) 13 | VERSION:=$(if $(TAG_NAME),$(TAG_NAME),$(SHA)) 14 | 15 | help: ## Display this help message 16 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' 17 | 18 | clean: ## Clean build output 19 | @echo BIN_OUTPUT: ${BIN_OUTPUT} 20 | rm -rf bin/ cover.out 21 | 22 | install-tools: ## Installs tool dependencies 23 | @cat tools.go | grep _ | awk -F'"' '{print $$2}' | xargs -tI % go install % 24 | 25 | test: ## Run tests 26 | CGO_ENABLED=1 go test -v -race -count=1 ./... 27 | 28 | test-cover: ## Run tests with coverage 29 | CGO_ENABLED=1 go test -v -race -count=1 -covermode=atomic -coverprofile=coverage.out ./... 30 | 31 | generate: ## Generate generates code using go:generate statements in source 32 | go generate ./... 33 | 34 | build: clean generate ## Build binary 35 | @echo VERSION: $(VERSION) 36 | go build -v -trimpath -ldflags '-X "main.version=${VERSION}"' -o ${BIN_OUTPUT} ${MAIN_DIRECTORY} 37 | 38 | outdated: ## Checks for outdated dependencies 39 | go list -u -m -json all | go-mod-outdated -update 40 | 41 | fmt: ## formats all *.go files added to git 42 | gofmt -s -l -w $(SRCS) 43 | 44 | coverage-out: test-cover ## Show coverage in cli 45 | @echo Coverage details 46 | @go tool cover -func=coverage.out 47 | 48 | coverage-html: test-cover ## Show coverage in browser 49 | @go tool cover -html=coverage.out 50 | 51 | dockerize: ## Created development docker image 52 | docker build -t philipssoftware/tabia:dev . 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tabia 2 | 3 | [![Go CI](https://github.com/philips-labs/tabia/workflows/Go%20CI/badge.svg)](https://github.com/philips-labs/tabia/actions) 4 | [![codecov](https://codecov.io/gh/philips-labs/tabia/branch/develop/graph/badge.svg?token=K2R9WOXNBm)](https://codecov.io/gh/philips-labs/tabia) 5 | 6 | Tabia means characteristic in Swahili. Tabia is giving us insights on the characteristics of our code bases. 7 | 8 | ## Setup 9 | 10 | Copy `.env.example` to `.env` and fill out the bitbucket token. This environment variable is read by the CLI and tests. Also vscode will read the variable when running tests or starting debugger. 11 | 12 | ```bash 13 | cp .env.example .env 14 | source .env 15 | env | grep TABIA 16 | ``` 17 | 18 | Install required tools: 19 | 20 | ```bash 21 | make install-tools 22 | ``` 23 | 24 | Add your $GO_PATH/bin folder to your path to make the installed Go tools globally available. 25 | 26 | ```bash 27 | export GO_PATH="$HOME/go" 28 | export PATH="$GO_PATH/bin:$PATH" 29 | ``` 30 | 31 | ## Build 32 | 33 | To build the CLI you can make use of the `build` target using `make`. 34 | 35 | ```bash 36 | make build 37 | ``` 38 | 39 | ## Test 40 | 41 | To run tests you can make use of the `test` target using `make`. 42 | 43 | ```bash 44 | make test 45 | ``` 46 | 47 | ## Run 48 | 49 | ### Bitbucket 50 | 51 | To interact with Bitbucket `tabia` makes use of the [Bitbucket 1.0 Rest API](https://docs.atlassian.com/bitbucket-server/rest/7.3.0/bitbucket-rest.html). 52 | 53 | ```bash 54 | bin/tabia bitbucket --help 55 | bin/tabia bitbucket projects --help 56 | bin/tabia bitbucket repositories --help 57 | ``` 58 | 59 | ### Github 60 | 61 | To interact with Github `tabia` makes use of the [Github graphql API](https://api.github.com/graphql). 62 | 63 | ```bash 64 | bin/tabia github --help 65 | bin/tabia github repositories --help 66 | ``` 67 | 68 | ### Output - Grimoirelab 69 | 70 | To expose the repositories in [Grimoirelab projects.json](https://github.com/chaoss/grimoirelab-sirmordred#projectsjson-) format, you can optionally provide a json file to map repositories to projects. By default the project will be mapped to the owner of the repository. Anything not matching the rules will fall back to this default. 71 | 72 | E.g.: 73 | 74 | ```bash 75 | bin/tabia -O philips-labs -M github-projects.json -F grimoirelab > projects.json 76 | ``` 77 | 78 | Regexes should be defined in the [following format](https://golang.org/pkg/regexp/syntax/). 79 | 80 | ```json 81 | { 82 | "rules": { 83 | "One Codebase": { "url": "tabia|varys|garo|^code\\-chars$" }, 84 | "HSDP": { "url": "(?i)hsdp" }, 85 | "iX": { "url": "(?i)ix\\-" }, 86 | "Licensing Entitlement": { "url": "(?i)lem\\-" }, 87 | "Code Signing": { "url": "(?i)^code\\-signing$|notary" } 88 | } 89 | } 90 | ``` 91 | 92 | #### Output - using template 93 | 94 | To generate the output for example in a markdown format you can use the option for a templated output format. This requires you to provide the path to a template file as well. Templates can be defined using the following [template/text package syntax](https://golang.org/pkg/text/template/). 95 | 96 | E.g.: 97 | 98 | ```md markdown.tmpl 99 | # Our repositories 100 | 101 | Our repository overview. Private/Internal repositories are marked with a __*__ 102 | 103 | {{range .}}* [{{ .Name}}]({{ .URL }}) {{if .IsPrivate() }}__*__{{end}} 104 | {{end}} 105 | ``` 106 | 107 | Using above template we can now easily generate a markdown file with this unordered list of repository links. 108 | 109 | ```bash 110 | bin/tabia -O philips-labs -F templated -T markdown.tmpl > repositories.md 111 | ``` 112 | 113 | #### Filter 114 | 115 | ##### Repositories 116 | 117 | The following repository fields can be filtered on. 118 | 119 | * ID 120 | * Name 121 | * Description 122 | * URL 123 | * SSHURL 124 | * Owner 125 | * Visibility 126 | * CreatedAt 127 | * UpdatedAt 128 | * PushedAt 129 | * Topics 130 | 131 | The following functions are available. 132 | 133 | * `func (RepositoryFilterEnv) Contains(s, substr string) bool` 134 | * `func (Repository) IsPublic() bool` 135 | * `func (Repository) IsInternal() bool` 136 | * `func (Repository) IsPrivate() bool` 137 | * `func (Repository) HasTopic(topic string) bool` 138 | * `func (Repository) CreatedSince(date string) bool` 139 | * `func (Repository) UpdatedSince(date string) bool` 140 | * `func (Repository) PushedSince(date string) bool` 141 | 142 | ```bash 143 | $ bin/tabia github repositories -O philips-labs -f '{ !.IsPrivate() && !.IsInternal() && !Contains(.Name, "terraform") }' 144 | Name Owner Visibility Clone 145 | 0001 helm2cf philips-labs Public https://github.com/philips-labs/helm2cf 146 | 0002 dct-notary-admin philips-labs Public https://github.com/philips-labs/dct-notary-admin 147 | 0003 notary philips-labs Public https://github.com/philips-labs/notary 148 | 0004 about-this-organization philips-labs Public https://github.com/philips-labs/about-this-organization 149 | 0005 sonar-scanner-action philips-labs Public https://github.com/philips-labs/sonar-scanner-action 150 | 0006 medical-delivery-drone philips-labs Public https://github.com/philips-labs/medical-delivery-drone 151 | 0007 dangerous-dave philips-labs Public https://github.com/philips-labs/dangerous-dave 152 | 0008 varys philips-labs Public https://github.com/philips-labs/varys 153 | 0009 garo philips-labs Public https://github.com/philips-labs/garo 154 | .......... 155 | ........... 156 | ........ 157 | ``` 158 | 159 | ##### Members 160 | 161 | The following member fields can be filtered on. 162 | 163 | * ID 164 | * Login 165 | * Name 166 | * Organization 167 | * SamlIdentity 168 | * ID 169 | 170 | The following functions are available. 171 | 172 | * `func (MemberFilterEnv) Contains(s, substr string) bool` 173 | 174 | #### Download contents 175 | 176 | ```bash 177 | bin/tabia github contents --repo philips-labs/tabia --file README.md --output downloads/tabia/README.md 178 | $ cat downloads/tabia/README.md 179 | # Tabia 180 | 181 | [![Go CI](https://github.com/philips-labs/tabia/workflows/Go%20CI/badge.svg)](https://github.com/philips-labs/tabia/actions) 182 | [![codecov](https://codecov.io/gh/philips-labs/tabia/branch/develop/graph/badge.svg?token=K2R9WOXNBm)](https://codecov.io/gh/philips-labs/tabia) 183 | ... 184 | ... 185 | .. 186 | ``` 187 | -------------------------------------------------------------------------------- /cmd/cmd.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import "github.com/urfave/cli/v2" 4 | 5 | // CreateCommands Creates CLI commands. 6 | func CreateCommands() []*cli.Command { 7 | return []*cli.Command{ 8 | createBitbucket(), 9 | createGithub(), 10 | createGitlab(), 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /cmd/cmd_bitbucket.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "html/template" 6 | "io" 7 | "os" 8 | "text/tabwriter" 9 | 10 | "github.com/urfave/cli/v2" 11 | 12 | "github.com/philips-labs/tabia/lib/bitbucket" 13 | "github.com/philips-labs/tabia/lib/grimoirelab" 14 | "github.com/philips-labs/tabia/lib/output" 15 | ) 16 | 17 | func createBitbucket() *cli.Command { 18 | return &cli.Command{ 19 | Name: "bitbucket", 20 | Usage: "Gets you some insight in Bitbucket repositories", 21 | Flags: []cli.Flag{ 22 | &cli.StringFlag{ 23 | Name: "api", 24 | Usage: "The api enpoint `ENDPOINT`", 25 | DefaultText: "https://bitbucket.atlas.philips.com/rest/api/1.0", 26 | EnvVars: []string{"TABIA_BITBUCKET_API"}, 27 | Required: true, 28 | }, 29 | &cli.StringFlag{ 30 | Name: "token", 31 | Aliases: []string{"t"}, 32 | Usage: "Calls the api using the given `TOKEN`", 33 | EnvVars: []string{"TABIA_BITBUCKET_TOKEN"}, 34 | Required: true, 35 | }, 36 | &cli.BoolFlag{ 37 | Name: "verbose", 38 | Usage: "Adds verbose logging", 39 | }, 40 | }, 41 | Subcommands: []*cli.Command{ 42 | { 43 | Name: "projects", 44 | Usage: "display insights on projects", 45 | Action: bitbucketProjects, 46 | Flags: []cli.Flag{ 47 | &cli.StringFlag{ 48 | Name: "format", 49 | Aliases: []string{"F"}, 50 | Usage: "Formats output in the given `FORMAT`", 51 | EnvVars: []string{"TABIA_OUTPUT_FORMAT"}, 52 | DefaultText: "", 53 | }}, 54 | }, 55 | { 56 | Name: "repositories", 57 | Usage: "display insights on repositories", 58 | Action: bitbucketRepositories, 59 | Flags: []cli.Flag{ 60 | &cli.BoolFlag{ 61 | Name: "all", 62 | Usage: "fetches repositories for all projects", 63 | }, 64 | &cli.StringSliceFlag{ 65 | Name: "projects", 66 | Aliases: []string{"P"}, 67 | Usage: "fetches repositories for given projects", 68 | }, 69 | &cli.StringFlag{ 70 | Name: "format", 71 | Aliases: []string{"F"}, 72 | Usage: "Formats output in the given `FORMAT`", 73 | EnvVars: []string{"TABIA_OUTPUT_FORMAT"}, 74 | DefaultText: "", 75 | }, 76 | &cli.PathFlag{ 77 | Name: "template", 78 | Aliases: []string{"T"}, 79 | Usage: "Formats output using the given `TEMPLATE`", 80 | TakesFile: true, 81 | }, 82 | }, 83 | }, 84 | }, 85 | } 86 | } 87 | 88 | func newBitbucketClient(c *cli.Context) *bitbucket.Client { 89 | api := c.String("api") 90 | verbose := c.Bool("verbose") 91 | token := c.String("token") 92 | 93 | var bbWriter io.Writer 94 | if verbose { 95 | bbWriter = c.App.Writer 96 | } 97 | return bitbucket.NewClientWithTokenAuth(api, token, bbWriter) 98 | } 99 | 100 | func bitbucketProjects(c *cli.Context) error { 101 | format := c.String("format") 102 | 103 | bb := newBitbucketClient(c) 104 | projects := make([]bitbucket.Project, 0) 105 | page := 0 106 | for { 107 | resp, err := bb.Projects.List(page) 108 | if err != nil { 109 | return err 110 | } 111 | projects = append(projects, resp.Values...) 112 | page = resp.NextPageStart 113 | if resp.IsLastPage { 114 | break 115 | } 116 | } 117 | 118 | switch format { 119 | case "json": 120 | err := output.PrintJSON(c.App.Writer, projects) 121 | if err != nil { 122 | return err 123 | } 124 | default: 125 | w := tabwriter.NewWriter(c.App.Writer, 3, 0, 2, ' ', tabwriter.TabIndent) 126 | fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", "ID", "Key", "Name", "Public") 127 | for _, project := range projects { 128 | fmt.Fprintf(w, "%d\t%s\t%s\t%t\n", project.ID, project.Key, project.Name, project.Public) 129 | } 130 | w.Flush() 131 | } 132 | 133 | return nil 134 | } 135 | 136 | func bitbucketRepositories(c *cli.Context) error { 137 | format := c.String("format") 138 | projects := c.StringSlice("projects") 139 | 140 | bb := newBitbucketClient(c) 141 | 142 | results := make([]bitbucket.Repository, 0) 143 | for _, project := range projects { 144 | resp, err := bb.Repositories.List(project) 145 | if err != nil { 146 | return err 147 | } 148 | results = append(results, resp.Values...) 149 | } 150 | 151 | switch format { 152 | case "json": 153 | err := output.PrintJSON(c.App.Writer, results) 154 | if err != nil { 155 | return err 156 | } 157 | case "grimoirelab": 158 | projects := grimoirelab.ConvertBitbucketToProjectsJSON(results, func(repo bitbucket.Repository) grimoirelab.Metadata { 159 | return grimoirelab.Metadata{ 160 | "title": repo.Project.Name, 161 | "program": "One Codebase", 162 | } 163 | }) 164 | err := output.PrintJSON(c.App.Writer, projects) 165 | if err != nil { 166 | return err 167 | } 168 | case "templated": 169 | if !c.IsSet("template") { 170 | return fmt.Errorf("you must specify the path to the template") 171 | } 172 | 173 | templateFile := c.Path("template") 174 | tmplContent, err := os.ReadFile(templateFile) 175 | if err != nil { 176 | return err 177 | } 178 | tmpl, err := template.New("repositories").Parse(string(tmplContent)) 179 | if err != nil { 180 | return err 181 | } 182 | err = tmpl.Execute(c.App.Writer, results) 183 | if err != nil { 184 | return err 185 | } 186 | default: 187 | w := tabwriter.NewWriter(c.App.Writer, 3, 0, 2, ' ', tabwriter.TabIndent) 188 | fmt.Fprintln(w, "Project\tID\tSlug\tName\tPublic\tClone") 189 | for _, repo := range results { 190 | httpClone := getCloneURL(repo.Links.Clone, "http") 191 | fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%t\t%s\n", repo.Project.Key, repo.ID, repo.Slug, repo.Name, repo.Public, httpClone) 192 | } 193 | w.Flush() 194 | } 195 | 196 | return nil 197 | } 198 | 199 | func getCloneURL(links []bitbucket.CloneLink, linkName string) string { 200 | for _, l := range links { 201 | if l.Name == linkName { 202 | return l.Href 203 | } 204 | } 205 | return "" 206 | } 207 | -------------------------------------------------------------------------------- /cmd/cmd_github.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | "text/tabwriter" 11 | 12 | "github.com/urfave/cli/v2" 13 | 14 | "github.com/philips-labs/tabia/lib/github" 15 | "github.com/philips-labs/tabia/lib/grimoirelab" 16 | "github.com/philips-labs/tabia/lib/output" 17 | ) 18 | 19 | func createGithub() *cli.Command { 20 | return &cli.Command{ 21 | Name: "github", 22 | Usage: "Gets you some insight in Github repositories", 23 | Flags: []cli.Flag{ 24 | &cli.StringFlag{ 25 | Name: "token", 26 | Aliases: []string{"t"}, 27 | Usage: "Calls the api using the given `TOKEN`", 28 | EnvVars: []string{"TABIA_GITHUB_TOKEN"}, 29 | Required: true, 30 | }, 31 | &cli.BoolFlag{ 32 | Name: "verbose", 33 | Usage: "Adds verbose logging", 34 | }, 35 | }, 36 | Subcommands: []*cli.Command{ 37 | { 38 | Name: "repositories", 39 | Usage: "display insights on repositories", 40 | Action: githubRepositories, 41 | Flags: []cli.Flag{ 42 | &cli.StringSliceFlag{ 43 | Name: "owner", 44 | Aliases: []string{"O"}, 45 | Usage: "fetches repositories for given owner", 46 | }, 47 | &cli.PathFlag{ 48 | Name: "matching", 49 | Aliases: []string{"M"}, 50 | Usage: "matches repositories to projects based on json file", 51 | TakesFile: true, 52 | }, 53 | &cli.StringFlag{ 54 | Name: "format", 55 | Aliases: []string{"F"}, 56 | Usage: "Formats output in the given `FORMAT`", 57 | EnvVars: []string{"TABIA_OUTPUT_FORMAT"}, 58 | DefaultText: "", 59 | }, 60 | &cli.PathFlag{ 61 | Name: "template", 62 | Aliases: []string{"T"}, 63 | Usage: "Formats output using the given `TEMPLATE`", 64 | TakesFile: true, 65 | }, 66 | &cli.StringFlag{ 67 | Name: "filter", 68 | Aliases: []string{"f"}, 69 | Usage: "filters repositories based on the given `EXPRESSION`", 70 | }, 71 | }, 72 | }, 73 | { 74 | Name: "members", 75 | Usage: "display insights on members", 76 | Action: githubMembers, 77 | Flags: []cli.Flag{ 78 | &cli.StringSliceFlag{ 79 | Name: "organization", 80 | Aliases: []string{"O"}, 81 | Usage: "fetches members for given organization", 82 | }, 83 | &cli.StringFlag{ 84 | Name: "format", 85 | Aliases: []string{"F"}, 86 | Usage: "Formats output in the given `FORMAT`", 87 | EnvVars: []string{"TABIA_OUTPUT_FORMAT"}, 88 | DefaultText: "", 89 | }, 90 | &cli.PathFlag{ 91 | Name: "template", 92 | Aliases: []string{"T"}, 93 | Usage: "Formats output using the given `TEMPLATE`", 94 | TakesFile: true, 95 | }, 96 | &cli.StringFlag{ 97 | Name: "filter", 98 | Aliases: []string{"f"}, 99 | Usage: "filters members based on the given `EXPRESSION`", 100 | }, 101 | }, 102 | }, 103 | { 104 | Name: "contents", 105 | Usage: "Gets contents from a repository", 106 | Action: githubContents, 107 | ArgsUsage: " ", 108 | Flags: []cli.Flag{ 109 | &cli.StringFlag{ 110 | Name: "repo", 111 | Aliases: []string{"R"}, 112 | Usage: "fetches content of given `REPO`", 113 | Required: true, 114 | }, 115 | &cli.StringFlag{ 116 | Name: "file", 117 | Aliases: []string{"f"}, 118 | Usage: "fetches content of given `FILE`", 119 | Required: true, 120 | }, 121 | &cli.StringFlag{ 122 | Name: "output", 123 | Aliases: []string{"o"}, 124 | Usage: "writes contents to `FILEPATH`", 125 | Required: false, 126 | }, 127 | }, 128 | }, 129 | }, 130 | } 131 | } 132 | 133 | func newGithubClient(c *cli.Context) *github.Client { 134 | verbose := c.Bool("verbose") 135 | token := c.String("token") 136 | 137 | var ghWriter io.Writer 138 | if verbose { 139 | ghWriter = c.App.Writer 140 | } 141 | 142 | return github.NewClientWithTokenAuth(token, ghWriter) 143 | } 144 | 145 | func githubMembers(c *cli.Context) error { 146 | owners := c.StringSlice("organization") 147 | format := c.String("format") 148 | filter := c.String("filter") 149 | 150 | client := newGithubClient(c) 151 | ctx, cancel := context.WithCancel(c.Context) 152 | defer cancel() 153 | 154 | var ghMembers []github.Member 155 | for _, owner := range owners { 156 | members, err := client.FetchOrganziationMembers(ctx, "royal-philips", owner) 157 | if err != nil { 158 | return err 159 | } 160 | filtered, err := github.ReduceMembers(members, filter) 161 | if err != nil { 162 | return err 163 | } 164 | ghMembers = append(ghMembers, filtered...) 165 | } 166 | 167 | switch format { 168 | case "json": 169 | err := output.PrintJSON(c.App.Writer, ghMembers) 170 | if err != nil { 171 | return err 172 | } 173 | case "templated": 174 | if !c.IsSet("template") { 175 | return fmt.Errorf("you must specify the path to the template") 176 | } 177 | 178 | templateFile := c.Path("template") 179 | tmplContent, err := os.ReadFile(templateFile) 180 | if err != nil { 181 | return err 182 | } 183 | err = output.PrintUsingTemplate(c.App.Writer, tmplContent, ghMembers) 184 | if err != nil { 185 | return err 186 | } 187 | default: 188 | w := tabwriter.NewWriter(c.App.Writer, 3, 0, 2, ' ', tabwriter.TabIndent) 189 | fmt.Fprintln(w, " \tLogin\tSAML Identity\tOrganization\tName") 190 | for i, m := range ghMembers { 191 | fmt.Fprintf(w, "%04d\t%s\t%s\t%s\t%s\n", i+1, m.Login, m.SamlIdentity.ID, m.Organization, m.Name) 192 | } 193 | w.Flush() 194 | } 195 | 196 | return nil 197 | } 198 | 199 | func githubRepositories(c *cli.Context) error { 200 | owners := c.StringSlice("owner") 201 | format := c.String("format") 202 | filter := c.String("filter") 203 | 204 | client := newGithubClient(c) 205 | ctx, cancel := context.WithCancel(c.Context) 206 | defer cancel() 207 | 208 | var repositories []github.Repository 209 | for _, owner := range owners { 210 | repos, err := client.FetchOrganizationRepositories(ctx, owner) 211 | if err != nil { 212 | return err 213 | } 214 | filtered, err := github.ReduceRepositories(repos, filter) 215 | if err != nil { 216 | return err 217 | } 218 | repositories = append(repositories, filtered...) 219 | } 220 | 221 | switch format { 222 | case "json": 223 | err := output.PrintJSON(c.App.Writer, repositories) 224 | if err != nil { 225 | return err 226 | } 227 | case "grimoirelab": 228 | projectMatchingConfig := c.Path("matching") 229 | json, err := os.Open(projectMatchingConfig) 230 | if err != nil { 231 | return err 232 | } 233 | defer json.Close() 234 | projectMatcher, err := grimoirelab.NewGithubProjectMatcherFromJSON(json) 235 | if err != nil { 236 | return err 237 | } 238 | 239 | projects := grimoirelab.ConvertGithubToProjectsJSON( 240 | repositories, 241 | func(repo github.Repository) grimoirelab.Metadata { 242 | return grimoirelab.Metadata{ 243 | "title": repo.Owner, 244 | "program": "One Codebase", 245 | } 246 | }, 247 | projectMatcher) 248 | err = output.PrintJSON(c.App.Writer, projects) 249 | if err != nil { 250 | return err 251 | } 252 | case "templated": 253 | if !c.IsSet("template") { 254 | return fmt.Errorf("you must specify the path to the template") 255 | } 256 | 257 | templateFile := c.Path("template") 258 | tmplContent, err := os.ReadFile(templateFile) 259 | if err != nil { 260 | return err 261 | } 262 | err = output.PrintUsingTemplate(c.App.Writer, tmplContent, repositories) 263 | if err != nil { 264 | return err 265 | } 266 | default: 267 | w := tabwriter.NewWriter(c.App.Writer, 3, 0, 2, ' ', tabwriter.TabIndent) 268 | fmt.Fprintln(w, " \tName\tOwner\tVisibility\tClone") 269 | for i, repo := range repositories { 270 | fmt.Fprintf(w, "%04d\t%s\t%s\t%s\t%s\n", i+1, repo.Name, repo.Owner, repo.Visibility, repo.URL) 271 | } 272 | w.Flush() 273 | } 274 | 275 | return nil 276 | } 277 | 278 | func githubContents(c *cli.Context) error { 279 | repo := c.String("repo") 280 | filePath := c.String("file") 281 | output := c.Path("output") 282 | 283 | client := newGithubClient(c) 284 | ctx, cancel := context.WithCancel(c.Context) 285 | defer cancel() 286 | 287 | repoParts := strings.Split(repo, "/") 288 | owner := repoParts[0] 289 | repo = repoParts[1] 290 | 291 | contents, err := client.DownloadContents(ctx, owner, repo, filePath) 292 | if err != nil { 293 | return err 294 | } 295 | 296 | if output != "" { 297 | if strings.Contains(output, "/") { 298 | dir := filepath.Dir(output) 299 | err := os.MkdirAll(dir, 0755) 300 | if err != nil { 301 | return err 302 | } 303 | } 304 | err := os.WriteFile(output, contents, 0644) 305 | if err != nil { 306 | return err 307 | } 308 | } else { 309 | fmt.Fprintf(c.App.Writer, "%s", contents) 310 | } 311 | 312 | return nil 313 | } 314 | -------------------------------------------------------------------------------- /cmd/cmd_gitlab.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "text/tabwriter" 9 | 10 | "github.com/urfave/cli/v2" 11 | 12 | "github.com/philips-labs/tabia/lib/gitlab" 13 | "github.com/philips-labs/tabia/lib/output" 14 | ) 15 | 16 | func createGitlab() *cli.Command { 17 | return &cli.Command{ 18 | Name: "gitlab", 19 | Usage: "Gets you some insight in Gitlab repositories", 20 | Flags: []cli.Flag{ 21 | &cli.StringFlag{ 22 | Name: "token", 23 | Aliases: []string{"t"}, 24 | Usage: "Calls the api using the given `TOKEN`", 25 | EnvVars: []string{"TABIA_GITLAB_TOKEN"}, 26 | Required: true, 27 | }, 28 | &cli.StringFlag{ 29 | Name: "instance", 30 | Usage: "The instance url `URL`", 31 | DefaultText: "https://gitlab.com/", 32 | EnvVars: []string{"TABIA_GITLAB_INSTANCE"}, 33 | Required: true, 34 | }, 35 | &cli.BoolFlag{ 36 | Name: "verbose", 37 | Usage: "Adds verbose logging", 38 | }, 39 | }, 40 | Subcommands: []*cli.Command{ 41 | { 42 | Name: "repositories", 43 | Usage: "display insights on repositories", 44 | Action: gitlabRepositories, 45 | Flags: []cli.Flag{ 46 | &cli.StringFlag{ 47 | Name: "format", 48 | Aliases: []string{"F"}, 49 | Usage: "Formats output in the given `FORMAT`", 50 | EnvVars: []string{"TABIA_OUTPUT_FORMAT"}, 51 | DefaultText: "", 52 | }, 53 | &cli.PathFlag{ 54 | Name: "template", 55 | Aliases: []string{"T"}, 56 | Usage: "Formats output using the given `TEMPLATE`", 57 | TakesFile: true, 58 | }, 59 | &cli.StringFlag{ 60 | Name: "filter", 61 | Aliases: []string{"f"}, 62 | Usage: "filters repositories based on the given `EXPRESSION`", 63 | }, 64 | }, 65 | }, 66 | }, 67 | } 68 | } 69 | 70 | func newGitlabClient(c *cli.Context) (*gitlab.Client, error) { 71 | instance := c.String("instance") 72 | verbose := c.Bool("verbose") 73 | token := c.String("token") 74 | 75 | var ghWriter io.Writer 76 | if verbose { 77 | ghWriter = c.App.Writer 78 | } 79 | 80 | return gitlab.NewClientWithTokenAuth(instance, token, ghWriter) 81 | } 82 | 83 | func gitlabRepositories(c *cli.Context) error { 84 | format := c.String("format") 85 | filter := c.String("filter") 86 | 87 | client, err := newGitlabClient(c) 88 | if err != nil { 89 | return err 90 | } 91 | ctx, cancel := context.WithCancel(c.Context) 92 | defer cancel() 93 | 94 | filters := gitlab.ConvertFiltersToListProjectOptions(filter) 95 | repos, err := client.ListRepositories(ctx, filters...) 96 | if err != nil { 97 | return err 98 | } 99 | 100 | filtered, err := gitlab.Reduce(repos, filter) 101 | if err != nil { 102 | return err 103 | } 104 | 105 | switch format { 106 | case "json": 107 | err := output.PrintJSON(c.App.Writer, filtered) 108 | if err != nil { 109 | return err 110 | } 111 | case "templated": 112 | if !c.IsSet("template") { 113 | return fmt.Errorf("you must specify the path to the template") 114 | } 115 | 116 | templateFile := c.Path("template") 117 | tmplContent, err := ioutil.ReadFile(templateFile) 118 | if err != nil { 119 | return err 120 | } 121 | err = output.PrintUsingTemplate(c.App.Writer, tmplContent, filtered) 122 | if err != nil { 123 | return err 124 | } 125 | default: 126 | w := tabwriter.NewWriter(c.App.Writer, 3, 0, 2, ' ', tabwriter.TabIndent) 127 | fmt.Fprintln(w, " \tID\tOwner\tName\tVisibility\tURL") 128 | for i, repo := range filtered { 129 | fmt.Fprintf(w, "%04d\t%d\t%s\t%s\t%s\t%s\n", i+1, repo.ID, repo.Owner, repo.Name, repo.Visibility, repo.URL) 130 | } 131 | w.Flush() 132 | } 133 | 134 | return nil 135 | } 136 | -------------------------------------------------------------------------------- /cmd/tabia/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "runtime" 8 | 9 | "github.com/urfave/cli/v2" 10 | 11 | "github.com/philips-labs/tabia/cmd" 12 | ) 13 | 14 | const ( 15 | appName = "tabia" 16 | ) 17 | 18 | var ( 19 | version = "dev" 20 | ) 21 | 22 | func main() { 23 | app := cli.NewApp() 24 | app.Name = appName 25 | app.HelpName = appName 26 | app.Usage = "code characteristics insights" 27 | app.EnableBashCompletion = true 28 | app.Version = version 29 | 30 | cli.VersionPrinter = func(c *cli.Context) { 31 | fmt.Printf("%s version %s %s/%s\n", appName, app.Version, runtime.GOOS, runtime.GOARCH) 32 | } 33 | 34 | app.Commands = cmd.CreateCommands() 35 | 36 | err := app.Run(os.Args) 37 | if err != nil { 38 | log.Fatal(err) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/philips-labs/tabia 2 | 3 | go 1.22.0 4 | toolchain go1.24.1 5 | 6 | require ( 7 | github.com/antonmedv/expr v1.15.5 8 | github.com/google/go-github/v33 v33.0.0 9 | github.com/goreleaser/goreleaser v1.26.2 10 | github.com/hashicorp/go-cleanhttp v0.5.2 11 | github.com/shurcooL/githubv4 v0.0.0-20220115235240-a14260e6f8a2 12 | github.com/stretchr/testify v1.10.0 13 | github.com/urfave/cli/v2 v2.27.6 14 | github.com/xanzy/go-gitlab v0.105.0 15 | golang.org/x/net v0.39.0 16 | golang.org/x/oauth2 v0.29.0 17 | golang.org/x/tools v0.32.0 18 | ) 19 | 20 | require ( 21 | cloud.google.com/go v0.112.1 // indirect 22 | cloud.google.com/go/compute/metadata v0.3.0 // indirect 23 | cloud.google.com/go/iam v1.1.6 // indirect 24 | cloud.google.com/go/kms v1.15.8 // indirect 25 | cloud.google.com/go/storage v1.39.1 // indirect 26 | code.gitea.io/sdk/gitea v0.18.0 // indirect 27 | dario.cat/mergo v1.0.0 // indirect 28 | github.com/AlekSi/pointer v1.2.0 // indirect 29 | github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect 30 | github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect 31 | github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 // indirect 32 | github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect 33 | github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 // indirect 34 | github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect 35 | github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 // indirect 36 | github.com/Azure/go-autorest v14.2.0+incompatible // indirect 37 | github.com/Azure/go-autorest/autorest v0.11.29 // indirect 38 | github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect 39 | github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect 40 | github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect 41 | github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect 42 | github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect 43 | github.com/Azure/go-autorest/logger v0.2.1 // indirect 44 | github.com/Azure/go-autorest/tracing v0.6.0 // indirect 45 | github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect 46 | github.com/BurntSushi/toml v1.4.0 // indirect 47 | github.com/Masterminds/goutils v1.1.1 // indirect 48 | github.com/Masterminds/semver/v3 v3.2.1 // indirect 49 | github.com/Masterminds/sprig/v3 v3.2.3 // indirect 50 | github.com/Microsoft/go-winio v0.6.1 // indirect 51 | github.com/ProtonMail/go-crypto v1.1.3 // indirect 52 | github.com/alessio/shellescape v1.4.1 // indirect 53 | github.com/anchore/bubbly v0.0.0-20230518153401-87b6af8ccf22 // indirect 54 | github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a // indirect 55 | github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb // indirect 56 | github.com/anchore/quill v0.4.1 // indirect 57 | github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect 58 | github.com/atc0005/go-teams-notify/v2 v2.10.0 // indirect 59 | github.com/aws/aws-sdk-go v1.53.0 // indirect 60 | github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect 61 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect 62 | github.com/aws/aws-sdk-go-v2/config v1.27.13 // indirect 63 | github.com/aws/aws-sdk-go-v2/credentials v1.17.13 // indirect 64 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect 65 | github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9 // indirect 66 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect 67 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect 68 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect 69 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 // indirect 70 | github.com/aws/aws-sdk-go-v2/service/ecr v1.28.0 // indirect 71 | github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.23.5 // indirect 72 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect 73 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 // indirect 74 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect 75 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 // indirect 76 | github.com/aws/aws-sdk-go-v2/service/kms v1.30.0 // indirect 77 | github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4 // indirect 78 | github.com/aws/aws-sdk-go-v2/service/sso v1.20.6 // indirect 79 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.0 // indirect 80 | github.com/aws/aws-sdk-go-v2/service/sts v1.28.7 // indirect 81 | github.com/aws/smithy-go v1.20.2 // indirect 82 | github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240514230400-03fa26f5508f // indirect 83 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect 84 | github.com/bahlo/generic-list-go v0.2.0 // indirect 85 | github.com/blacktop/go-dwarf v1.0.9 // indirect 86 | github.com/blacktop/go-macho v1.1.162 // indirect 87 | github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb // indirect 88 | github.com/bluesky-social/indigo v0.0.0-20240411170459-440932307e0d // indirect 89 | github.com/buger/jsonparser v1.1.1 // indirect 90 | github.com/caarlos0/ctrlc v1.2.0 // indirect 91 | github.com/caarlos0/env/v11 v11.0.1 // indirect 92 | github.com/caarlos0/go-reddit/v3 v3.0.1 // indirect 93 | github.com/caarlos0/go-shellwords v1.0.12 // indirect 94 | github.com/caarlos0/go-version v0.1.1 // indirect 95 | github.com/caarlos0/log v0.4.4 // indirect 96 | github.com/carlmjohnson/versioninfo v0.22.5 // indirect 97 | github.com/cavaliergopher/cpio v1.0.1 // indirect 98 | github.com/cenkalti/backoff/v4 v4.2.1 // indirect 99 | github.com/charmbracelet/bubbletea v0.22.1 // indirect 100 | github.com/charmbracelet/lipgloss v0.10.0 // indirect 101 | github.com/charmbracelet/x/exp/ordered v0.0.0-20231010190216-1cb11efc897d // indirect 102 | github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect 103 | github.com/cloudflare/circl v1.3.8 // indirect 104 | github.com/containerd/console v1.0.4 // indirect 105 | github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect 106 | github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect 107 | github.com/cyphar/filepath-securejoin v0.2.5 // indirect 108 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 109 | github.com/davidmz/go-pageant v1.0.2 // indirect 110 | github.com/dghubble/go-twitter v0.0.0-20211115160449-93a8679adecb // indirect 111 | github.com/dghubble/oauth1 v0.7.3 // indirect 112 | github.com/dghubble/sling v1.4.0 // indirect 113 | github.com/dimchansky/utfbom v1.1.1 // indirect 114 | github.com/distribution/reference v0.5.0 // indirect 115 | github.com/docker/cli v25.0.4+incompatible // indirect 116 | github.com/docker/distribution v2.8.3+incompatible // indirect 117 | github.com/docker/docker v26.1.5+incompatible // indirect 118 | github.com/docker/docker-credential-helpers v0.8.1 // indirect 119 | github.com/docker/go-connections v0.4.0 // indirect 120 | github.com/docker/go-units v0.5.0 // indirect 121 | github.com/dustin/go-humanize v1.0.1 // indirect 122 | github.com/elliotchance/orderedmap/v2 v2.2.0 // indirect 123 | github.com/emirpasic/gods v1.18.1 // indirect 124 | github.com/evanphx/json-patch/v5 v5.6.0 // indirect 125 | github.com/felixge/httpsnoop v1.0.4 // indirect 126 | github.com/fsnotify/fsnotify v1.7.0 // indirect 127 | github.com/gabriel-vasile/mimetype v1.4.2 // indirect 128 | github.com/github/smimesign v0.2.0 // indirect 129 | github.com/go-fed/httpsig v1.1.0 // indirect 130 | github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect 131 | github.com/go-git/go-billy/v5 v5.6.0 // indirect 132 | github.com/go-git/go-git/v5 v5.13.0 // indirect 133 | github.com/go-logr/logr v1.4.1 // indirect 134 | github.com/go-logr/stdr v1.2.2 // indirect 135 | github.com/go-openapi/analysis v0.23.0 // indirect 136 | github.com/go-openapi/errors v0.22.0 // indirect 137 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 138 | github.com/go-openapi/jsonreference v0.21.0 // indirect 139 | github.com/go-openapi/loads v0.22.0 // indirect 140 | github.com/go-openapi/runtime v0.28.0 // indirect 141 | github.com/go-openapi/spec v0.21.0 // indirect 142 | github.com/go-openapi/strfmt v0.23.0 // indirect 143 | github.com/go-openapi/swag v0.23.0 // indirect 144 | github.com/go-openapi/validate v0.24.0 // indirect 145 | github.com/go-restruct/restruct v1.2.0-alpha // indirect 146 | github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 // indirect 147 | github.com/gobwas/glob v0.2.3 // indirect 148 | github.com/gogo/protobuf v1.3.2 // indirect 149 | github.com/golang-jwt/jwt/v4 v4.5.2 // indirect 150 | github.com/golang-jwt/jwt/v5 v5.2.2 // indirect 151 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 152 | github.com/golang/protobuf v1.5.4 // indirect 153 | github.com/google/go-containerregistry v0.19.1 // indirect 154 | github.com/google/go-github/v62 v62.0.0 // indirect 155 | github.com/google/go-querystring v1.1.0 // indirect 156 | github.com/google/ko v0.15.4 // indirect 157 | github.com/google/rpmpack v0.6.1-0.20240329070804-c2247cbb881a // indirect 158 | github.com/google/s2a-go v0.1.7 // indirect 159 | github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect 160 | github.com/google/uuid v1.6.0 // indirect 161 | github.com/google/wire v0.6.0 // indirect 162 | github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect 163 | github.com/googleapis/gax-go/v2 v2.12.3 // indirect 164 | github.com/goreleaser/chglog v0.6.1 // indirect 165 | github.com/goreleaser/fileglob v1.3.0 // indirect 166 | github.com/goreleaser/nfpm/v2 v2.37.1 // indirect 167 | github.com/gorilla/websocket v1.5.1 // indirect 168 | github.com/hashicorp/errwrap v1.1.0 // indirect 169 | github.com/hashicorp/go-multierror v1.1.1 // indirect 170 | github.com/hashicorp/go-retryablehttp v0.7.7 // indirect 171 | github.com/hashicorp/go-version v1.6.0 // indirect 172 | github.com/hashicorp/golang-lru v1.0.2 // indirect 173 | github.com/hashicorp/hcl v1.0.1-vault-5 // indirect 174 | github.com/huandu/xstrings v1.3.3 // indirect 175 | github.com/imdario/mergo v0.3.16 // indirect 176 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 177 | github.com/invopop/jsonschema v0.12.0 // indirect 178 | github.com/ipfs/bbloom v0.0.4 // indirect 179 | github.com/ipfs/go-block-format v0.2.0 // indirect 180 | github.com/ipfs/go-cid v0.4.1 // indirect 181 | github.com/ipfs/go-datastore v0.6.0 // indirect 182 | github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect 183 | github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect 184 | github.com/ipfs/go-ipfs-util v0.0.3 // indirect 185 | github.com/ipfs/go-ipld-cbor v0.1.0 // indirect 186 | github.com/ipfs/go-ipld-format v0.6.0 // indirect 187 | github.com/ipfs/go-log v1.0.5 // indirect 188 | github.com/ipfs/go-log/v2 v2.5.1 // indirect 189 | github.com/ipfs/go-metrics-interface v0.0.1 // indirect 190 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect 191 | github.com/jbenet/goprocess v0.1.4 // indirect 192 | github.com/jmespath/go-jmespath v0.4.0 // indirect 193 | github.com/josharian/intern v1.0.0 // indirect 194 | github.com/kevinburke/ssh_config v1.2.0 // indirect 195 | github.com/klauspost/compress v1.17.8 // indirect 196 | github.com/klauspost/cpuid/v2 v2.2.7 // indirect 197 | github.com/klauspost/pgzip v1.2.6 // indirect 198 | github.com/kylelemons/godebug v1.1.0 // indirect 199 | github.com/letsencrypt/boulder v0.0.0-20231026200631-000cd05d5491 // indirect 200 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 201 | github.com/magiconair/properties v1.8.7 // indirect 202 | github.com/mailru/easyjson v0.7.7 // indirect 203 | github.com/mattn/go-isatty v0.0.20 // indirect 204 | github.com/mattn/go-localereader v0.0.1 // indirect 205 | github.com/mattn/go-mastodon v0.0.8 // indirect 206 | github.com/mattn/go-runewidth v0.0.15 // indirect 207 | github.com/minio/sha256-simd v1.0.1 // indirect 208 | github.com/mitchellh/copystructure v1.2.0 // indirect 209 | github.com/mitchellh/go-homedir v1.1.0 // indirect 210 | github.com/mitchellh/mapstructure v1.5.0 // indirect 211 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 212 | github.com/moby/docker-image-spec v1.3.1 // indirect 213 | github.com/mr-tron/base58 v1.2.0 // indirect 214 | github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect 215 | github.com/muesli/cancelreader v0.2.2 // indirect 216 | github.com/muesli/mango v0.1.0 // indirect 217 | github.com/muesli/mango-cobra v1.2.0 // indirect 218 | github.com/muesli/mango-pflag v0.1.0 // indirect 219 | github.com/muesli/reflow v0.3.0 // indirect 220 | github.com/muesli/roff v0.1.0 // indirect 221 | github.com/muesli/termenv v0.15.2 // indirect 222 | github.com/multiformats/go-base32 v0.1.0 // indirect 223 | github.com/multiformats/go-base36 v0.2.0 // indirect 224 | github.com/multiformats/go-multibase v0.2.0 // indirect 225 | github.com/multiformats/go-multihash v0.2.3 // indirect 226 | github.com/multiformats/go-varint v0.0.7 // indirect 227 | github.com/oklog/ulid v1.3.1 // indirect 228 | github.com/opencontainers/go-digest v1.0.0 // indirect 229 | github.com/opencontainers/image-spec v1.1.0 // indirect 230 | github.com/opentracing/opentracing-go v1.2.0 // indirect 231 | github.com/pelletier/go-toml v1.9.5 // indirect 232 | github.com/pelletier/go-toml/v2 v2.1.0 // indirect 233 | github.com/pjbgf/sha1cd v0.3.0 // indirect 234 | github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect 235 | github.com/pkg/errors v0.9.1 // indirect 236 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 237 | github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect 238 | github.com/rivo/uniseg v0.4.7 // indirect 239 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 240 | github.com/sagikazarmark/locafero v0.4.0 // indirect 241 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 242 | github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect 243 | github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect 244 | github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect 245 | github.com/shopspring/decimal v1.2.0 // indirect 246 | github.com/shurcooL/graphql v0.0.0-20200928012149-18c5c3165e3a // indirect 247 | github.com/sigstore/cosign/v2 v2.2.4 // indirect 248 | github.com/sigstore/rekor v1.3.6 // indirect 249 | github.com/sigstore/sigstore v1.8.3 // indirect 250 | github.com/sirupsen/logrus v1.9.3 // indirect 251 | github.com/skeema/knownhosts v1.3.0 // indirect 252 | github.com/slack-go/slack v0.13.0 // indirect 253 | github.com/sourcegraph/conc v0.3.0 // indirect 254 | github.com/spaolacci/murmur3 v1.1.0 // indirect 255 | github.com/spf13/afero v1.11.0 // indirect 256 | github.com/spf13/cast v1.6.0 // indirect 257 | github.com/spf13/cobra v1.8.0 // indirect 258 | github.com/spf13/pflag v1.0.5 // indirect 259 | github.com/spf13/viper v1.18.2 // indirect 260 | github.com/subosito/gotenv v1.6.0 // indirect 261 | github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect 262 | github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect 263 | github.com/ulikunitz/xz v0.5.12 // indirect 264 | github.com/vbatts/tar-split v0.11.5 // indirect 265 | github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651 // indirect 266 | github.com/wagoodman/go-progress v0.0.0-20220614130704-4b1c25a33c7c // indirect 267 | github.com/whyrusleeping/cbor-gen v0.1.1-0.20240311221002-68b9f235c302 // indirect 268 | github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect 269 | github.com/xanzy/ssh-agent v0.3.3 // indirect 270 | github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect 271 | gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect 272 | go.mongodb.org/mongo-driver v1.14.0 // indirect 273 | go.opencensus.io v0.24.0 // indirect 274 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect 275 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect 276 | go.opentelemetry.io/otel v1.24.0 // indirect 277 | go.opentelemetry.io/otel/metric v1.24.0 // indirect 278 | go.opentelemetry.io/otel/trace v1.24.0 // indirect 279 | go.uber.org/atomic v1.11.0 // indirect 280 | go.uber.org/automaxprocs v1.5.3 // indirect 281 | go.uber.org/multierr v1.11.0 // indirect 282 | go.uber.org/zap v1.27.0 // indirect 283 | gocloud.dev v0.37.0 // indirect 284 | golang.org/x/crypto v0.37.0 // indirect 285 | golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect 286 | golang.org/x/mod v0.24.0 // indirect 287 | golang.org/x/sync v0.13.0 // indirect 288 | golang.org/x/sys v0.32.0 // indirect 289 | golang.org/x/term v0.31.0 // indirect 290 | golang.org/x/text v0.24.0 // indirect 291 | golang.org/x/time v0.5.0 // indirect 292 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect 293 | google.golang.org/api v0.172.0 // indirect 294 | google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect 295 | google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect 296 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect 297 | google.golang.org/grpc v1.62.1 // indirect 298 | google.golang.org/protobuf v1.33.0 // indirect 299 | gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect 300 | gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect 301 | gopkg.in/ini.v1 v1.67.0 // indirect 302 | gopkg.in/mail.v2 v2.3.1 // indirect 303 | gopkg.in/warnings.v0 v0.1.2 // indirect 304 | gopkg.in/yaml.v3 v3.0.1 // indirect 305 | gotest.tools/v3 v3.1.0 // indirect 306 | lukechampine.com/blake3 v1.2.1 // indirect 307 | sigs.k8s.io/kind v0.23.0 // indirect 308 | sigs.k8s.io/yaml v1.4.0 // indirect 309 | software.sslmate.com/src/go-pkcs12 v0.4.0 // indirect 310 | ) 311 | -------------------------------------------------------------------------------- /lib/bitbucket/bitbucket.go: -------------------------------------------------------------------------------- 1 | package bitbucket 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | 9 | "golang.org/x/net/proxy" 10 | 11 | "github.com/philips-labs/tabia/lib/transport" 12 | ) 13 | 14 | type PagedResponse struct { 15 | Size int `json:"size"` 16 | Limit int `json:"limit"` 17 | Start int `json:"start"` 18 | IsLastPage bool `json:"isLastPage"` 19 | NextPageStart int `json:"nextPageStart"` 20 | } 21 | 22 | type Links struct { 23 | Clone []CloneLink `json:"clone,omitempty"` 24 | Self *[]struct { 25 | Href string `json:"href"` 26 | } `json:"self,omitempty"` 27 | } 28 | 29 | type CloneLink struct { 30 | Href string `json:"href"` 31 | Name string `json:"name"` 32 | } 33 | 34 | type Client struct { 35 | baseEndpoint string 36 | Auth interface{} 37 | HttpClient *http.Client 38 | Projects Projects 39 | Repositories Repositories 40 | } 41 | 42 | type BasicAuth struct { 43 | Username string 44 | Password string 45 | } 46 | 47 | type TokenAuth struct { 48 | Token string 49 | } 50 | 51 | func NewClientWithBasicAuth(endpoint, username, password string, writer io.Writer) *Client { 52 | httpClient := new(http.Client) 53 | if writer != nil { 54 | httpClient.Transport = transport.TeeRoundTripper{ 55 | RoundTripper: new(http.Transport), 56 | Writer: writer, 57 | } 58 | } 59 | c := &Client{ 60 | baseEndpoint: endpoint, 61 | Auth: BasicAuth{Username: username, Password: password}, 62 | HttpClient: httpClient, 63 | } 64 | 65 | c.Projects = Projects{c} 66 | c.Repositories = Repositories{c} 67 | return c 68 | } 69 | 70 | func NewClientWithTokenAuth(endpoint, token string, writer io.Writer) *Client { 71 | httpClient := new(http.Client) 72 | if writer != nil { 73 | httpClient.Transport = transport.TeeRoundTripper{ 74 | RoundTripper: new(http.Transport), 75 | Writer: writer, 76 | } 77 | } 78 | c := &Client{ 79 | baseEndpoint: endpoint, 80 | Auth: TokenAuth{Token: token}, 81 | HttpClient: httpClient, 82 | } 83 | c.Projects = Projects{c} 84 | c.Repositories = Repositories{c} 85 | return c 86 | } 87 | 88 | func (c *Client) SetSocksProxy(url string) error { 89 | dialer, err := proxy.SOCKS5("tcp", url, nil, proxy.Direct) 90 | if err != nil { 91 | return err 92 | } 93 | httpTransport := &http.Transport{} 94 | httpTransport.Dial = dialer.Dial 95 | c.HttpClient.Transport = httpTransport 96 | 97 | return nil 98 | } 99 | 100 | func (c *Client) RawRequest(method, url, text string) (io.ReadCloser, error) { 101 | // body := strings.NewReader(text) 102 | 103 | req, err := http.NewRequest(method, url, nil) 104 | if err != nil { 105 | return nil, err 106 | } 107 | if text != "" { 108 | req.Header.Set("Content-Type", "application/json") 109 | } 110 | 111 | err = c.authenticateRequest(req) 112 | if err != nil { 113 | return nil, err 114 | } 115 | 116 | return c.doRawRequest(req, false) 117 | } 118 | 119 | func (c *Client) authenticateRequest(req *http.Request) error { 120 | switch auth := c.Auth.(type) { 121 | case BasicAuth: 122 | req.SetBasicAuth(auth.Username, auth.Password) 123 | case TokenAuth: 124 | req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", auth.Token)) 125 | default: 126 | return errors.New("unsupported authentication method") 127 | } 128 | 129 | return nil 130 | } 131 | 132 | func (c *Client) doRawRequest(req *http.Request, emtpyResponse bool) (io.ReadCloser, error) { 133 | resp, err := c.HttpClient.Do(req) 134 | if err != nil { 135 | return nil, err 136 | } 137 | if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { 138 | resp.Body.Close() 139 | return nil, errors.New(resp.Status) 140 | } 141 | 142 | if emtpyResponse { 143 | resp.Body.Close() 144 | return nil, nil 145 | } 146 | 147 | if resp.Body == nil { 148 | resp.Body.Close() 149 | return nil, errors.New("response body is nil") 150 | } 151 | 152 | return resp.Body, nil 153 | } 154 | -------------------------------------------------------------------------------- /lib/bitbucket/bitbucket_test.go: -------------------------------------------------------------------------------- 1 | package bitbucket_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net" 7 | "net/http" 8 | "net/http/httptest" 9 | "os" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/assert" 14 | 15 | "github.com/philips-labs/tabia/lib/bitbucket" 16 | ) 17 | 18 | func bitbucketTestClient(handler http.Handler) (*bitbucket.Client, string, func()) { 19 | s := httptest.NewServer(handler) 20 | baseUrl := s.Listener.Addr().String() 21 | apiUrl := "http://" + baseUrl + "/rest/api/1.0" 22 | token := os.Getenv("TABIA_BITBUCKET_TOKEN") 23 | bb := bitbucket.NewClientWithTokenAuth(apiUrl, token, nil) 24 | bb.HttpClient.Transport = &http.Transport{ 25 | DialContext: func(_ context.Context, network, _ string) (net.Conn, error) { 26 | return net.Dial(network, baseUrl) 27 | }, 28 | } 29 | 30 | return bb, apiUrl, s.Close 31 | } 32 | 33 | func TestClientWithTokenAuth(t *testing.T) { 34 | if len(os.Getenv("TABIA_BITBUCKET_TOKEN")) == 0 { 35 | t.Skip("skipping integration test, depending on environment variable") 36 | } 37 | assert := assert.New(t) 38 | 39 | s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 40 | fmt.Fprintln(w, `{ "name": "My repo" }`) 41 | })) 42 | baseUrl := s.Listener.Addr().String() 43 | apiUrl := "http://" + baseUrl + "/rest/api/1.0" 44 | token := "asd12bjkhu23uy12iu3hh" 45 | project := "philips-internal" 46 | 47 | var writer strings.Builder 48 | bb := bitbucket.NewClientWithTokenAuth(apiUrl, token, &writer) 49 | 50 | assert.Equal(bitbucket.TokenAuth{Token: token}, bb.Auth) 51 | 52 | _, err := bb.Repositories.List(project) 53 | assert.NoError(err) 54 | assert.NotEmpty(writer) 55 | assert.Equal(fmt.Sprintf("GET: %s/projects/%s/repos?limit=100 ", apiUrl, project), writer.String()) 56 | } 57 | 58 | func TestClientWithBasicAuth(t *testing.T) { 59 | if len(os.Getenv("TABIA_BITBUCKET_TOKEN")) == 0 { 60 | t.Skip("skipping integration test, depending on environment variable") 61 | } 62 | assert := assert.New(t) 63 | 64 | s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 65 | fmt.Fprintln(w, `{ "name": "My repo" }`) 66 | })) 67 | baseUrl := s.Listener.Addr().String() 68 | apiUrl := "http://" + baseUrl + "/rest/api/1.0" 69 | user := "johndoe" 70 | pass := "S3cr3t!" 71 | project := "philips-internal" 72 | var writer strings.Builder 73 | bb := bitbucket.NewClientWithBasicAuth(apiUrl, user, pass, &writer) 74 | 75 | assert.Equal(bitbucket.BasicAuth{Username: user, Password: pass}, bb.Auth) 76 | 77 | _, err := bb.Repositories.List(project) 78 | assert.NoError(err) 79 | assert.NotEmpty(writer) 80 | assert.Equal(fmt.Sprintf("GET: %s/projects/%s/repos?limit=100 ", apiUrl, project), writer.String()) 81 | } 82 | -------------------------------------------------------------------------------- /lib/bitbucket/projects.go: -------------------------------------------------------------------------------- 1 | package bitbucket 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | ) 8 | 9 | type Projects struct { 10 | c *Client 11 | } 12 | 13 | type Project struct { 14 | Key string `json:"key"` 15 | ID int `json:"id"` 16 | Name string `json:"name"` 17 | Description string `json:"description"` 18 | Public bool `json:"public"` 19 | Type string `json:"type"` 20 | Links Links `json:"links"` 21 | } 22 | 23 | type ProjectsResponse struct { 24 | *PagedResponse 25 | Values []Project `json:"values"` 26 | } 27 | 28 | func (r *Projects) List(start int) (*ProjectsResponse, error) { 29 | url := fmt.Sprintf("%s/projects?limit=100&start=%d", r.c.baseEndpoint, start) 30 | body, err := r.c.RawRequest("GET", url, "") 31 | if err != nil { 32 | return nil, err 33 | } 34 | defer body.Close() 35 | 36 | return decodeProjects(body) 37 | } 38 | 39 | func decodeProjects(body io.ReadCloser) (*ProjectsResponse, error) { 40 | var result ProjectsResponse 41 | 42 | if err := json.NewDecoder(body).Decode(&result); err != nil { 43 | return nil, fmt.Errorf("failed to read projects json: %w", err) 44 | } 45 | 46 | return &result, nil 47 | } 48 | -------------------------------------------------------------------------------- /lib/bitbucket/projects_test.go: -------------------------------------------------------------------------------- 1 | package bitbucket_test 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "net/http" 7 | "testing" 8 | 9 | "github.com/philips-labs/tabia/lib/bitbucket" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | var stubProjectsResponse = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 15 | projectsResponse := bitbucket.ProjectsResponse{ 16 | Values: make([]bitbucket.Project, 100), 17 | } 18 | resp, _ := json.Marshal(projectsResponse) 19 | _, _ = w.Write(resp) 20 | }) 21 | 22 | func TestListProjectsRaw(t *testing.T) { 23 | assert := assert.New(t) 24 | bb, apiBaseURL, teardown := bitbucketTestClient(stubProjectsResponse) 25 | defer teardown() 26 | 27 | resp, err := bb.RawRequest("GET", apiBaseURL+"/projects", "") 28 | if !assert.NoError(err) { 29 | return 30 | } 31 | defer resp.Close() 32 | 33 | assert.NotNil(resp) 34 | bytes, err := io.ReadAll(resp) 35 | assert.NoError(err) 36 | assert.NotEmpty(bytes) 37 | } 38 | 39 | func TestListProjects(t *testing.T) { 40 | assert := assert.New(t) 41 | 42 | bb, _, teardown := bitbucketTestClient(stubProjectsResponse) 43 | defer teardown() 44 | 45 | resp, err := bb.Projects.List(0) 46 | 47 | if !assert.NoError(err) { 48 | return 49 | } 50 | 51 | assert.NotNil(resp) 52 | assert.Len(resp.Values, 100) 53 | } 54 | -------------------------------------------------------------------------------- /lib/bitbucket/repositories.go: -------------------------------------------------------------------------------- 1 | package bitbucket 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | ) 8 | 9 | type Repositories struct { 10 | c *Client 11 | } 12 | 13 | type Repository struct { 14 | Slug string `json:"slug"` 15 | ID int `json:"id"` 16 | Name string `json:"name"` 17 | Description string `json:"description"` 18 | ScmID string `json:"scmId"` 19 | State string `json:"state"` 20 | StatusMessage string `json:"statusMessage"` 21 | Forkable bool `json:"forkable"` 22 | Project Project `json:"project"` 23 | Public bool `json:"public"` 24 | Links Links `json:"links"` 25 | } 26 | 27 | type RepositoriesResponse struct { 28 | *PagedResponse 29 | Values []Repository `json:"values"` 30 | } 31 | 32 | func (r *Repositories) List(project string) (*RepositoriesResponse, error) { 33 | url := fmt.Sprintf("%s/projects/%s/repos?limit=100", r.c.baseEndpoint, project) 34 | repos, err := r.c.RawRequest("GET", url, "") 35 | if err != nil { 36 | return nil, err 37 | } 38 | defer repos.Close() 39 | 40 | return decodeRepos(repos) 41 | } 42 | 43 | func decodeRepos(body io.ReadCloser) (*RepositoriesResponse, error) { 44 | var result RepositoriesResponse 45 | if err := json.NewDecoder(body).Decode(&result); err != nil { 46 | return nil, fmt.Errorf("failed to read repositories json: %w", err) 47 | } 48 | 49 | return &result, nil 50 | } 51 | -------------------------------------------------------------------------------- /lib/bitbucket/repositories_test.go: -------------------------------------------------------------------------------- 1 | package bitbucket_test 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "net/http" 7 | "testing" 8 | 9 | "github.com/philips-labs/tabia/lib/bitbucket" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | var stubRepositoriesResponse = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 15 | projectsResponse := bitbucket.RepositoriesResponse{ 16 | Values: make([]bitbucket.Repository, 12), 17 | } 18 | resp, _ := json.Marshal(projectsResponse) 19 | _, _ = w.Write(resp) 20 | }) 21 | 22 | func TestListRepositoriesRaw(t *testing.T) { 23 | assert := assert.New(t) 24 | 25 | bb, apiBaseURL, teardown := bitbucketTestClient(stubRepositoriesResponse) 26 | defer teardown() 27 | resp, err := bb.RawRequest("GET", apiBaseURL+"/projects/VID/repos", "") 28 | if !assert.NoError(err) { 29 | return 30 | } 31 | defer resp.Close() 32 | 33 | assert.NotNil(resp) 34 | bytes, err := io.ReadAll(resp) 35 | assert.NoError(err) 36 | assert.NotEmpty(bytes) 37 | } 38 | 39 | func TestListRepositories(t *testing.T) { 40 | assert := assert.New(t) 41 | 42 | bb, _, teardown := bitbucketTestClient(stubRepositoriesResponse) 43 | defer teardown() 44 | resp, err := bb.Repositories.List("ACE") 45 | 46 | if !assert.NoError(err) { 47 | return 48 | } 49 | 50 | assert.NotNil(resp) 51 | assert.Len(resp.Values, 12) 52 | } 53 | -------------------------------------------------------------------------------- /lib/github/client.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "net/http" 7 | 8 | "github.com/google/go-github/v33/github" 9 | "github.com/shurcooL/githubv4" 10 | "golang.org/x/oauth2" 11 | 12 | "github.com/philips-labs/tabia/lib/transport" 13 | ) 14 | 15 | type Client struct { 16 | httpClient *http.Client 17 | restClient *github.Client 18 | *githubv4.Client 19 | } 20 | 21 | func NewClientWithTokenAuth(token string, writer io.Writer) *Client { 22 | src := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) 23 | httpClient := oauth2.NewClient(context.Background(), src) 24 | if writer != nil { 25 | httpClient.Transport = transport.TeeRoundTripper{ 26 | RoundTripper: httpClient.Transport, 27 | Writer: writer, 28 | } 29 | } 30 | client := githubv4.NewClient(httpClient) 31 | restClient := github.NewClient(httpClient) 32 | 33 | return &Client{httpClient, restClient, client} 34 | } 35 | -------------------------------------------------------------------------------- /lib/github/client_test.go: -------------------------------------------------------------------------------- 1 | package github_test 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | 11 | "github.com/philips-labs/tabia/lib/github" 12 | ) 13 | 14 | func TestClient(t *testing.T) { 15 | if len(os.Getenv("TABIA_GITHUB_TOKEN")) == 0 { 16 | t.Skip("skipping integration test, depending on environment variable") 17 | } 18 | assert := assert.New(t) 19 | 20 | var buf strings.Builder 21 | 22 | client := github.NewClientWithTokenAuth(os.Getenv("TABIA_GITHUB_TOKEN"), &buf) 23 | var q struct{} 24 | err := client.Client.Query(context.Background(), q, nil) 25 | 26 | assert.EqualError(err, "Field must have selections (anonymous query returns Query but has no selections. Did you mean ' { ... }'?)") 27 | assert.NotEmpty(buf) 28 | assert.Equal("POST: https://api.github.com/graphql {\"query\":\"{}\"}\n", buf.String()) 29 | } 30 | -------------------------------------------------------------------------------- /lib/github/contents.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "io" 6 | ) 7 | 8 | // DownloadContents downloads file contents from the given filepath 9 | func (c *Client) DownloadContents(ctx context.Context, owner, repo, filepath string) ([]byte, error) { 10 | contents, _, err := c.restClient.Repositories.DownloadContents(ctx, owner, repo, filepath, nil) 11 | if err != nil { 12 | return nil, err 13 | } 14 | 15 | defer contents.Close() 16 | return io.ReadAll(contents) 17 | } 18 | -------------------------------------------------------------------------------- /lib/github/contents_test.go: -------------------------------------------------------------------------------- 1 | package github_test 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/philips-labs/tabia/lib/github" 11 | ) 12 | 13 | func TestDownloadContents(t *testing.T) { 14 | if len(os.Getenv("TABIA_GITHUB_TOKEN")) == 0 { 15 | t.Skip("skipping integration test, depending on environment variable") 16 | } 17 | assert := assert.New(t) 18 | gh := github.NewClientWithTokenAuth(os.Getenv("TABIA_GITHUB_TOKEN"), nil) 19 | contents, err := gh.DownloadContents(context.Background(), "philips-labs", "tabia", "README.md") 20 | if assert.NoError(err) { 21 | readme, _ := os.ReadFile("../../README.md") 22 | assert.NotEmpty(contents) 23 | assert.Equal(string(readme[:100]), string(contents[:100])) 24 | } 25 | 26 | contents, err = gh.DownloadContents(context.Background(), "philips-labs", "tabia", "IamNotThere.txt") 27 | if assert.Error(err) { 28 | assert.EqualError(err, "No file named IamNotThere.txt found in .") 29 | } 30 | assert.Empty(contents) 31 | } 32 | -------------------------------------------------------------------------------- /lib/github/filter.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | 8 | "github.com/antonmedv/expr" 9 | 10 | "github.com/philips-labs/tabia/lib/shared" 11 | ) 12 | 13 | // RepositoryFilterEnv filter environment for repositories 14 | type RepositoryFilterEnv struct { 15 | Repositories []Repository 16 | } 17 | 18 | // MemberFilterEnv filter environment for members 19 | type MemberFilterEnv struct { 20 | Members []Member 21 | } 22 | 23 | // Contains reports wether substring is in s. 24 | func (MemberFilterEnv) Contains(s, substring string) bool { 25 | return strings.Contains(s, substring) 26 | } 27 | 28 | // Contains reports wether substring is in s. 29 | func (RepositoryFilterEnv) Contains(s, substring string) bool { 30 | return strings.Contains(s, substring) 31 | } 32 | 33 | // IsPublic indicates if a repository has public visibility. 34 | func (r Repository) IsPublic() bool { 35 | return r.Visibility == shared.Public 36 | } 37 | 38 | // IsInternal indicates if a repository has internal visibility. 39 | func (r Repository) IsInternal() bool { 40 | return r.Visibility == shared.Internal 41 | } 42 | 43 | // IsPrivate indicates if a repository has private visibility. 44 | func (r Repository) IsPrivate() bool { 45 | return r.Visibility == shared.Private 46 | } 47 | 48 | // HasTopic indicates if a repository has a given topic. 49 | func (r Repository) HasTopic(topic string) bool { 50 | for _, t := range r.Topics { 51 | if strings.EqualFold(t.Name, topic) { 52 | return true 53 | } 54 | } 55 | 56 | return false 57 | } 58 | 59 | // HasLanguage indicates if a repository has a given language. 60 | func (r Repository) HasLanguage(language string) bool { 61 | for _, l := range r.Languages { 62 | if strings.EqualFold(l.Name, language) { 63 | return true 64 | } 65 | } 66 | 67 | return false 68 | } 69 | 70 | // UpdatedSince indicates if a repository has been updated since the given date. 71 | // Date has to be given in RFC3339 format, e.g. `2006-01-02T15:04:05Z07:00`. 72 | func (r Repository) UpdatedSince(date string) bool { 73 | return equalOrAfter(r.UpdatedAt, date) 74 | } 75 | 76 | // PushedSince indicates if a repository has been pushed since the given date. 77 | // Date has to be given in RFC3339 format, e.g. `2006-01-02T15:04:05Z07:00`. 78 | func (r Repository) PushedSince(date string) bool { 79 | return equalOrAfter(r.PushedAt, date) 80 | } 81 | 82 | // CreatedSince indicates if a repository has been created since the given date. 83 | // Date has to be given in RFC3339 format, e.g. `2006-01-02T15:04:05Z07:00`. 84 | func (r Repository) CreatedSince(date string) bool { 85 | return equalOrAfter(r.CreatedAt, date) 86 | } 87 | 88 | func equalOrAfter(a time.Time, date string) bool { 89 | since, err := time.Parse(time.RFC3339, date) 90 | if err != nil { 91 | return true 92 | } 93 | 94 | return a.Equal(since) || a.After(since) 95 | } 96 | 97 | // ReduceRepositories filters the repositories based on the given filter 98 | func ReduceRepositories(repositories []Repository, filter string) ([]Repository, error) { 99 | if strings.TrimSpace(filter) == "" { 100 | return repositories, nil 101 | } 102 | 103 | program, err := expr.Compile(fmt.Sprintf("filter(Repositories, %s)", filter)) 104 | if err != nil { 105 | return nil, err 106 | } 107 | 108 | result, err := expr.Run(program, RepositoryFilterEnv{repositories}) 109 | if err != nil { 110 | return nil, err 111 | } 112 | var repos []Repository 113 | for _, repo := range result.([]interface{}) { 114 | repos = append(repos, repo.(Repository)) 115 | } 116 | return repos, nil 117 | } 118 | 119 | // ReduceMembers filters the members based on the given filter 120 | func ReduceMembers(members []Member, filter string) ([]Member, error) { 121 | if strings.TrimSpace(filter) == "" { 122 | return members, nil 123 | } 124 | 125 | program, err := expr.Compile(fmt.Sprintf("filter(Members, %s)", filter)) 126 | if err != nil { 127 | return nil, err 128 | } 129 | 130 | result, err := expr.Run(program, MemberFilterEnv{members}) 131 | if err != nil { 132 | return nil, err 133 | } 134 | var filtered []Member 135 | for _, m := range result.([]interface{}) { 136 | filtered = append(filtered, m.(Member)) 137 | } 138 | return filtered, nil 139 | } 140 | -------------------------------------------------------------------------------- /lib/github/filter_test.go: -------------------------------------------------------------------------------- 1 | package github_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/philips-labs/tabia/lib/github" 11 | "github.com/philips-labs/tabia/lib/shared" 12 | ) 13 | 14 | func TestReduceRepositories(t *testing.T) { 15 | assert := assert.New(t) 16 | 17 | repos := []github.Repository{ 18 | { 19 | Name: "tabia", Visibility: shared.Public, Owner: "philips-labs", 20 | Languages: []github.Language{{Name: "Go"}}, 21 | CreatedAt: time.Now().Add(-24 * time.Hour), 22 | PushedAt: time.Now().Add(-24 * time.Hour), 23 | UpdatedAt: time.Now().Add(-24 * time.Hour), 24 | }, 25 | { 26 | Name: "garo", Visibility: shared.Public, Owner: "philips-labs", 27 | CreatedAt: time.Now().Add(-96 * time.Hour), 28 | PushedAt: time.Now().Add(-96 * time.Hour), 29 | UpdatedAt: time.Now().Add(-96 * time.Hour), 30 | }, 31 | { 32 | Name: "dct-notary-admin", Visibility: shared.Public, Owner: "philips-labs", 33 | CreatedAt: time.Now().Add(-24 * time.Hour), 34 | PushedAt: time.Now().Add(-24 * time.Hour), 35 | UpdatedAt: time.Now().Add(-24 * time.Hour), 36 | }, 37 | { 38 | Name: "company-draft", Visibility: shared.Internal, Owner: "philips-labs", 39 | CreatedAt: time.Now().Add(-48 * time.Hour), 40 | PushedAt: time.Now().Add(-48 * time.Hour), 41 | UpdatedAt: time.Now().Add(-48 * time.Hour), 42 | }, 43 | { 44 | Name: "top-secret", Visibility: shared.Private, Owner: "philips-labs", 45 | CreatedAt: time.Now().Add(-24 * time.Hour), 46 | PushedAt: time.Now().Add(-24 * time.Hour), 47 | UpdatedAt: time.Now().Add(-24 * time.Hour), 48 | Topics: []github.Topic{{Name: "ip"}}, 49 | }, 50 | } 51 | 52 | reduced, err := github.ReduceRepositories(repos, "") 53 | if assert.NoError(err) { 54 | assert.Len(reduced, 5) 55 | assert.ElementsMatch(reduced, repos) 56 | } 57 | 58 | reduced, err = github.ReduceRepositories(repos, `.Name == "garo"`) 59 | if assert.NoError(err) { 60 | assert.Len(reduced, 1) 61 | assert.Contains(reduced, repos[1]) 62 | } 63 | 64 | reduced, err = github.ReduceRepositories(repos, `{ .Name == "garo" }`) 65 | if assert.NoError(err) { 66 | assert.Len(reduced, 1) 67 | assert.Contains(reduced, repos[1]) 68 | } 69 | 70 | reduced, err = github.ReduceRepositories(repos, `{ .IsPublic() }`) 71 | if assert.NoError(err) { 72 | assert.Len(reduced, 3) 73 | assert.Contains(reduced, repos[0]) 74 | assert.Contains(reduced, repos[1]) 75 | assert.Contains(reduced, repos[2]) 76 | } 77 | 78 | reduced, err = github.ReduceRepositories(repos, `{ .IsInternal() }`) 79 | if assert.NoError(err) { 80 | assert.Len(reduced, 1) 81 | assert.Contains(reduced, repos[3]) 82 | } 83 | 84 | reduced, err = github.ReduceRepositories(repos, `{ .IsPrivate() }`) 85 | if assert.NoError(err) { 86 | assert.Len(reduced, 1) 87 | assert.Contains(reduced, repos[4]) 88 | } 89 | 90 | reduced, err = github.ReduceRepositories(repos, `{ Contains(.Name, "ar") }`) 91 | if assert.NoError(err) { 92 | assert.Len(reduced, 2) 93 | assert.Contains(reduced, repos[1]) 94 | assert.Contains(reduced, repos[2]) 95 | } 96 | 97 | reduced, err = github.ReduceRepositories(repos, `{ .HasTopic("ip") }`) 98 | if assert.NoError(err) { 99 | assert.Len(reduced, 1) 100 | assert.Contains(reduced, repos[4]) 101 | } 102 | 103 | reduced, err = github.ReduceRepositories(repos, `{ .HasLanguage("go") }`) 104 | if assert.NoError(err) { 105 | assert.Len(reduced, 1) 106 | assert.Contains(reduced, repos[0]) 107 | } 108 | 109 | since := time.Now().Add(-25 * time.Hour).Format(time.RFC3339) 110 | reduced, err = github.ReduceRepositories(repos, fmt.Sprintf(`{ .CreatedSince("%s") }`, since)) 111 | if assert.NoError(err) { 112 | assert.Len(reduced, 3) 113 | assert.Contains(reduced, repos[0]) 114 | assert.Contains(reduced, repos[2]) 115 | assert.Contains(reduced, repos[4]) 116 | } 117 | 118 | since = time.Now().Add(-97 * time.Hour).Format(time.RFC3339) 119 | reduced, err = github.ReduceRepositories(repos, fmt.Sprintf(`{ .UpdatedSince("%s") }`, since)) 120 | if assert.NoError(err) { 121 | assert.Len(reduced, 5) 122 | assert.Contains(reduced, repos[0]) 123 | assert.Contains(reduced, repos[1]) 124 | assert.Contains(reduced, repos[2]) 125 | assert.Contains(reduced, repos[3]) 126 | assert.Contains(reduced, repos[4]) 127 | } 128 | 129 | since = time.Now().Add(-49 * time.Hour).Format(time.RFC3339) 130 | reduced, err = github.ReduceRepositories(repos, fmt.Sprintf(`{ .PushedSince("%s") }`, since)) 131 | if assert.NoError(err) { 132 | assert.Len(reduced, 4) 133 | assert.Contains(reduced, repos[0]) 134 | assert.Contains(reduced, repos[2]) 135 | assert.Contains(reduced, repos[3]) 136 | assert.Contains(reduced, repos[4]) 137 | } 138 | } 139 | 140 | func TestReduceMembers(t *testing.T) { 141 | assert := assert.New(t) 142 | 143 | members := []github.Member{ 144 | {Name: "John Doe"}, 145 | {Name: "Marco Franssen"}, 146 | {Name: "Jane Doe"}, 147 | } 148 | 149 | reduced, err := github.ReduceMembers(members, `.Name == "Marco Franssen"`) 150 | if assert.NoError(err) { 151 | assert.Len(reduced, 1) 152 | assert.Contains(reduced, members[1]) 153 | } 154 | 155 | reduced, err = github.ReduceMembers(members, `{ Contains(.Name, "Doe") }`) 156 | if assert.NoError(err) { 157 | assert.Len(reduced, 2) 158 | assert.Contains(reduced, members[0]) 159 | assert.Contains(reduced, members[2]) 160 | } 161 | } 162 | 163 | func TestReduceWrongExpression(t *testing.T) { 164 | assert := assert.New(t) 165 | 166 | repos := []github.Repository{ 167 | {Name: "tabia", Visibility: shared.Public}, 168 | } 169 | 170 | reduced, err := github.ReduceRepositories(repos, `.Name = "tabia"`) 171 | assert.Error(err) 172 | assert.EqualError(err, "unexpected token Operator(\"=\") (1:28)\n | filter(Repositories, .Name = \"tabia\")\n | ...........................^") 173 | assert.Nil(reduced) 174 | 175 | reduced, err = github.ReduceRepositories(repos, `{ UnExistingFunc(.URL, "stuff") }`) 176 | assert.Error(err) 177 | assert.EqualError(err, "cannot fetch UnExistingFunc from github.RepositoryFilterEnv (1:24)\n | filter(Repositories, { UnExistingFunc(.URL, \"stuff\") })\n | .......................^") 178 | assert.Nil(reduced) 179 | } 180 | -------------------------------------------------------------------------------- /lib/github/graphql/graphql.go: -------------------------------------------------------------------------------- 1 | package graphql 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/shurcooL/githubv4" 7 | ) 8 | 9 | type PageInfo struct { 10 | HasNextPage bool `json:"has_next_page,omitempty"` 11 | EndCursor githubv4.String `json:"end_cursor,omitempty"` 12 | } 13 | 14 | type Owner struct { 15 | Login string `json:"login,omitempty"` 16 | } 17 | 18 | type Repository struct { 19 | ID string `json:"id,omitempty"` 20 | Name string `json:"name,omitempty"` 21 | Description string `json:"description,omitempty"` 22 | URL string `json:"url,omitempty"` 23 | SSHURL string `json:"ssh_url,omitempty"` 24 | Owner Owner `json:"owner,omitempty"` 25 | IsPrivate bool `json:"is_private,omitempty"` 26 | CreatedAt time.Time `json:"created_at,omitempty"` 27 | UpdatedAt time.Time `json:"updated_at,omitempty"` 28 | PushedAt time.Time `json:"pushed_at,omitempty"` 29 | RepositoryTopics RepositoryTopics `graphql:"repositoryTopics(first: 25)" json:"repository_topics,omitempty"` 30 | Collaborators Collaborators `graphql:"collaborators(first: 15, affiliation: DIRECT)" json:"collaborators,omitempty"` 31 | Languages Languages `graphql:"languages(first: 10, orderBy: {field: SIZE, direction: DESC})" json:"languages,omitempty"` 32 | ForkCount int `json:"fork_count,omitempty"` 33 | Stargazers StargazersConnection `graphql:"stargazers(first: 0)" json:"stargazers,omitempty"` 34 | Watchers UserConnection `graphql:"watchers(first: 0)" json:"watchers,omitempty"` 35 | } 36 | 37 | type Languages struct { 38 | Edges []LanguageEdge `json:"edges,omitempty"` 39 | } 40 | 41 | type LanguageEdge struct { 42 | Size int 43 | Node LanguageNode 44 | } 45 | 46 | type LanguageNode struct { 47 | Name string 48 | Color string 49 | } 50 | 51 | type RepositoryTopics struct { 52 | Nodes []RepositoryTopic `json:"nodes,omitempty"` 53 | } 54 | 55 | type RepositoryTopic struct { 56 | Topic Topic `json:"topic,omitempty"` 57 | ResourcePath string `json:"resource_path,omitempty"` 58 | } 59 | 60 | type Topic struct { 61 | Name string `json:"name,omitempty"` 62 | } 63 | 64 | type Repositories struct { 65 | TotalCount int `json:"total_count,omitempty"` 66 | PageInfo PageInfo `json:"page_info,omitempty"` 67 | Nodes []Repository `json:"nodes,omitempty"` 68 | } 69 | 70 | type Collaborators struct { 71 | TotalCount int `json:"total_count,omitempty"` 72 | PageInfo PageInfo `json:"page_info,omitempty"` 73 | Nodes []Collaborator `json:"nodes,omitempty"` 74 | } 75 | 76 | type Collaborator struct { 77 | Name string `json:"name,omitempty"` 78 | Login string `json:"login,omitempty"` 79 | AvatarURL string `json:"avatar_url,omitempty"` 80 | } 81 | 82 | type StargazersConnection struct { 83 | TotalCount int `json:"total_count,omitempty"` 84 | } 85 | 86 | type UserConnection struct { 87 | TotalCount int `json:"total_count,omitempty"` 88 | } 89 | 90 | type Organization struct { 91 | Repositories Repositories `graphql:"repositories(first: 100, after: $repoCursor)" json:"repositories,omitempty"` 92 | } 93 | -------------------------------------------------------------------------------- /lib/github/graphql/members.go: -------------------------------------------------------------------------------- 1 | package graphql 2 | 3 | type Enterprise struct { 4 | OwnerInfo OwnerInfo `json:"ownerInfo,omitempty"` 5 | } 6 | 7 | type OwnerInfo struct { 8 | SamlIdentityProvider SamlIdentityProvider `json:"samlIdentityProvider,omitempty"` 9 | } 10 | 11 | type SamlIdentityProvider struct { 12 | SsoURL string `json:"ssoUrl,omitempty"` 13 | ExternalIdentities ExternalIdentities `graphql:"externalIdentities(first: 100, after: $identityCursor)" json:"externalIdentities,omitempty"` 14 | } 15 | 16 | type ExternalIdentities struct { 17 | Edges []ExternalIdentityEdge `json:"edges,omitempty"` 18 | PageInfo PageInfo `json:"pageInfo,omitempty"` 19 | } 20 | 21 | type ExternalIdentityEdge struct { 22 | Node ExternalIdentityNode `json:"node,omitempty"` 23 | } 24 | 25 | type ExternalIdentityNode struct { 26 | SamlIdentity SamlIdentity `json:"samlIdentity,omitempty"` 27 | User Member `json:"user,omitempty"` 28 | } 29 | 30 | type SamlIdentity struct { 31 | NameId string `json:"nameId,omitempty"` 32 | Username string `json:"username,omitempty"` 33 | } 34 | 35 | type Member struct { 36 | ID string `json:"id,omitempty"` 37 | Login string `json:"login,omitempty"` 38 | Name string `json:"name,omitempty"` 39 | Email string `json:"email,omitempty"` 40 | Organization OrganizationName `graphql:"organization(login: $organization)" json:"organization,omitempty"` 41 | } 42 | 43 | type OrganizationName struct { 44 | Name string `json:"name,omitempty"` 45 | } 46 | -------------------------------------------------------------------------------- /lib/github/graphql/search.go: -------------------------------------------------------------------------------- 1 | package graphql 2 | 3 | type RepositorySearch struct { 4 | RepositoryCount int 5 | PageInfo PageInfo 6 | Edges []Edge 7 | } 8 | 9 | type Edge struct { 10 | Node Node 11 | } 12 | 13 | type Node struct { 14 | Repository Repository `graphql:"... on Repository"` 15 | } 16 | -------------------------------------------------------------------------------- /lib/github/members.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/shurcooL/githubv4" 7 | 8 | "github.com/philips-labs/tabia/lib/github/graphql" 9 | ) 10 | 11 | type Member struct { 12 | ID string `json:"id,omitempty"` 13 | Login string `json:"login,omitempty"` 14 | Name string `json:"name,omitempty"` 15 | Email string `json:"email,omitempty"` 16 | Organization string `json:"organization,omitempty"` 17 | SamlIdentity *SamlIdentity `json:"saml_identity,omitempty"` 18 | } 19 | 20 | type SamlIdentity struct { 21 | ID string `json:"id,omitempty"` 22 | } 23 | 24 | func (c *Client) FetchOrganziationMembers(ctx context.Context, enterprise, organization string) ([]Member, error) { 25 | var q struct { 26 | Enterprise graphql.Enterprise `graphql:"enterprise(slug: $enterprise)"` 27 | } 28 | 29 | variables := map[string]interface{}{ 30 | "enterprise": githubv4.String(enterprise), 31 | "organization": githubv4.String(organization), 32 | "identityCursor": (*githubv4.String)(nil), 33 | } 34 | 35 | var identities []graphql.ExternalIdentityNode 36 | for { 37 | err := c.Query(ctx, &q, variables) 38 | if err != nil { 39 | return nil, err 40 | } 41 | idp := q.Enterprise.OwnerInfo.SamlIdentityProvider 42 | identities = append(identities, identityEdges(idp.ExternalIdentities.Edges)...) 43 | if !idp.ExternalIdentities.PageInfo.HasNextPage { 44 | break 45 | } 46 | 47 | variables["identityCursor"] = githubv4.NewString(idp.ExternalIdentities.PageInfo.EndCursor) 48 | } 49 | 50 | return MapIdentitiesToMembers(identities), nil 51 | } 52 | 53 | func identityEdges(edges []graphql.ExternalIdentityEdge) []graphql.ExternalIdentityNode { 54 | var identities []graphql.ExternalIdentityNode 55 | for _, edge := range edges { 56 | // seems users without the organization field populated are 57 | // still returned by the api despite the filter on this field 58 | if edge.Node.User.Organization.Name != "" { 59 | identities = append(identities, edge.Node) 60 | } 61 | } 62 | return identities 63 | } 64 | 65 | func MapIdentitiesToMembers(identities []graphql.ExternalIdentityNode) []Member { 66 | members := make([]Member, len(identities)) 67 | for i, identity := range identities { 68 | members[i] = Member{ 69 | ID: identity.User.ID, 70 | Login: identity.User.Login, 71 | Name: identity.User.Name, 72 | Email: identity.User.Email, 73 | Organization: identity.User.Organization.Name, 74 | SamlIdentity: &SamlIdentity{ID: identity.SamlIdentity.NameId}, 75 | } 76 | } 77 | 78 | return members 79 | } 80 | -------------------------------------------------------------------------------- /lib/github/members_test.go: -------------------------------------------------------------------------------- 1 | package github_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/philips-labs/tabia/lib/github" 9 | "github.com/philips-labs/tabia/lib/github/graphql" 10 | ) 11 | 12 | func TestMapIdentitiesToMembers(t *testing.T) { 13 | assert := assert.New(t) 14 | 15 | graphqlIdentities := []graphql.ExternalIdentityNode{ 16 | { 17 | SamlIdentity: graphql.SamlIdentity{NameId: "ldapID1234", Username: "ldapID1234"}, 18 | User: graphql.Member{ID: "githubID1234", Login: "marcofranssen", Name: "Marco Franssen", Organization: struct { 19 | Name string `json:"name,omitempty"` 20 | }{Name: "philips-labs"}}}, 21 | { 22 | SamlIdentity: graphql.SamlIdentity{NameId: "ldapID5678", Username: "ldapID5678"}, 23 | User: graphql.Member{ID: "githubID5678", Login: "jdoe", Name: "John Doe", Organization: struct { 24 | Name string `json:"name,omitempty"` 25 | }{Name: "philips-labs"}}, 26 | }, 27 | } 28 | 29 | ghMembers := github.MapIdentitiesToMembers(graphqlIdentities) 30 | 31 | assert.Len(ghMembers, 2) 32 | assert.Equal(graphqlIdentities[0].User.ID, ghMembers[0].ID) 33 | assert.Equal(graphqlIdentities[1].User.ID, ghMembers[1].ID) 34 | 35 | assert.Equal(graphqlIdentities[0].User.Login, ghMembers[0].Login) 36 | assert.Equal(graphqlIdentities[1].User.Login, ghMembers[1].Login) 37 | 38 | assert.Equal(graphqlIdentities[0].User.Name, ghMembers[0].Name) 39 | assert.Equal(graphqlIdentities[1].User.Name, ghMembers[1].Name) 40 | 41 | assert.Equal(graphqlIdentities[0].User.Organization.Name, ghMembers[0].Organization) 42 | assert.Equal(graphqlIdentities[1].User.Organization.Name, ghMembers[1].Organization) 43 | 44 | assert.Equal(graphqlIdentities[0].SamlIdentity.NameId, ghMembers[0].SamlIdentity.ID) 45 | assert.Equal(graphqlIdentities[1].SamlIdentity.NameId, ghMembers[1].SamlIdentity.ID) 46 | } 47 | -------------------------------------------------------------------------------- /lib/github/repositories.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "time" 8 | 9 | "github.com/google/go-github/v33/github" 10 | "github.com/shurcooL/githubv4" 11 | 12 | "github.com/philips-labs/tabia/lib/github/graphql" 13 | "github.com/philips-labs/tabia/lib/shared" 14 | ) 15 | 16 | type RestRepo struct { 17 | Name string 18 | } 19 | 20 | type Repository struct { 21 | ID string `json:"id,omitempty"` 22 | Name string `json:"name,omitempty"` 23 | Description string `json:"description,omitempty"` 24 | URL string `json:"url,omitempty"` 25 | SSHURL string `json:"ssh_url,omitempty"` 26 | Owner string `json:"owner,omitempty"` 27 | Visibility shared.Visibility `json:"visibility"` 28 | CreatedAt time.Time `json:"created_at,omitempty"` 29 | UpdatedAt time.Time `json:"updated_at,omitempty"` 30 | PushedAt time.Time `json:"pushed_at,omitempty"` 31 | ForkCount int `json:"fork_count,omitempty"` 32 | StargazerCount int `json:"stargazer_count,omitempty"` 33 | WatcherCount int `json:"watcher_count,omitempty"` 34 | Topics []Topic `json:"topics,omitempty"` 35 | Languages []Language `json:"languages,omitempty"` 36 | Collaborators []Collaborator `json:"collaborators,omitempty"` 37 | } 38 | 39 | type Collaborator struct { 40 | *graphql.Collaborator 41 | } 42 | 43 | type Topic struct { 44 | Name string `json:"name,omitempty"` 45 | URL string `json:"url,omitempty"` 46 | } 47 | 48 | type Language struct { 49 | Name string `json:"name,omitempty"` 50 | Color string `json:"color,omitempty"` 51 | Size int `json:"size,omitempty"` 52 | } 53 | 54 | func (c *Client) FetchOrganizationRepositories(ctx context.Context, owner string) ([]Repository, error) { 55 | var q struct { 56 | Repositories graphql.RepositorySearch `graphql:"search(query: $query, type: REPOSITORY, first: 100, after: $repoCursor)"` 57 | } 58 | 59 | // archived repositories are filtered as they give error when fetching collaborators 60 | // this bug is known with Github. 61 | // Also see https://github.com/shurcooL/githubv4/issues/72, on proposal for better error handling 62 | variables := map[string]interface{}{ 63 | "query": githubv4.String(fmt.Sprintf("org:%s archived:false", owner)), 64 | "repoCursor": (*githubv4.String)(nil), 65 | } 66 | 67 | var repositories []graphql.Repository 68 | for { 69 | err := c.Query(ctx, &q, variables) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | repositories = append(repositories, repositoryEdges(q.Repositories.Edges)...) 75 | if !q.Repositories.PageInfo.HasNextPage { 76 | break 77 | } 78 | 79 | variables["repoCursor"] = githubv4.NewString(q.Repositories.PageInfo.EndCursor) 80 | } 81 | 82 | // currently the graphql api does not seem to support private vs internal. 83 | // therefore we use the rest api to fetch the private repos so we can determine private vs internal in the Map function. 84 | privateRepos, err := c.FetchRestRepositories(ctx, owner, "private") 85 | if err != nil { 86 | return nil, err 87 | } 88 | 89 | return Map(repositories, privateRepos) 90 | } 91 | 92 | func (c *Client) FetchRestRepositories(ctx context.Context, owner, repoType string) ([]*github.Repository, error) { 93 | var allRepos []*github.Repository 94 | 95 | opt := &github.RepositoryListByOrgOptions{Type: repoType} 96 | for { 97 | repos, resp, err := c.restClient.Repositories.ListByOrg(ctx, owner, opt) 98 | if err != nil { 99 | return nil, err 100 | } 101 | defer resp.Body.Close() 102 | allRepos = append(allRepos, repos...) 103 | if resp.NextPage == 0 { 104 | break 105 | } 106 | opt.Page = resp.NextPage 107 | } 108 | 109 | return allRepos, nil 110 | } 111 | 112 | func Map(repositories []graphql.Repository, privateRepositories []*github.Repository) ([]Repository, error) { 113 | repos := make([]Repository, len(repositories)) 114 | for i, repo := range repositories { 115 | repos[i] = Repository{ 116 | ID: repo.ID, 117 | Name: repo.Name, 118 | Description: strings.TrimSpace(repo.Description), 119 | URL: repo.URL, 120 | SSHURL: repo.SSHURL, 121 | Owner: repo.Owner.Login, 122 | CreatedAt: repo.CreatedAt, 123 | UpdatedAt: repo.UpdatedAt, 124 | PushedAt: repo.PushedAt, 125 | ForkCount: repo.ForkCount, 126 | StargazerCount: repo.Stargazers.TotalCount, 127 | WatcherCount: repo.Watchers.TotalCount, 128 | Topics: mapTopics(repo.RepositoryTopics), 129 | Languages: mapLanguages(repo.Languages), 130 | Collaborators: mapCollaborators(repo.Collaborators), 131 | } 132 | 133 | if repo.IsPrivate { 134 | isPrivate := false 135 | for _, privRepo := range privateRepositories { 136 | if *privRepo.Name == repo.Name { 137 | isPrivate = true 138 | break 139 | } 140 | } 141 | 142 | if isPrivate { 143 | repos[i].Visibility = shared.Private 144 | } else { 145 | repos[i].Visibility = shared.Internal 146 | } 147 | } else { 148 | repos[i].Visibility = shared.Public 149 | } 150 | } 151 | 152 | return repos, nil 153 | } 154 | 155 | func mapTopics(topics graphql.RepositoryTopics) []Topic { 156 | ghTopics := make([]Topic, len(topics.Nodes)) 157 | for i, topic := range topics.Nodes { 158 | ghTopics[i] = Topic{Name: topic.Topic.Name, URL: fmt.Sprintf("https://github.com%s", topic.ResourcePath)} 159 | } 160 | return ghTopics 161 | } 162 | 163 | func mapLanguages(languages graphql.Languages) []Language { 164 | ghLanguages := make([]Language, len(languages.Edges)) 165 | for i, lang := range languages.Edges { 166 | ghLanguages[i] = Language{Name: lang.Node.Name, Size: lang.Size, Color: lang.Node.Color} 167 | } 168 | return ghLanguages 169 | } 170 | 171 | func mapCollaborators(collaborators graphql.Collaborators) []Collaborator { 172 | ghCollaborators := make([]Collaborator, len(collaborators.Nodes)) 173 | for i := range collaborators.Nodes { 174 | ghCollaborators[i] = Collaborator{&collaborators.Nodes[i]} 175 | } 176 | return ghCollaborators 177 | } 178 | 179 | func repositoryEdges(edges []graphql.Edge) []graphql.Repository { 180 | var repositories []graphql.Repository 181 | for _, edge := range edges { 182 | repositories = append(repositories, edge.Node.Repository) 183 | } 184 | return repositories 185 | } 186 | -------------------------------------------------------------------------------- /lib/github/repositories_test.go: -------------------------------------------------------------------------------- 1 | package github_test 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | "testing" 8 | 9 | gh "github.com/google/go-github/v33/github" 10 | "github.com/stretchr/testify/assert" 11 | 12 | "github.com/philips-labs/tabia/lib/github" 13 | "github.com/philips-labs/tabia/lib/github/graphql" 14 | "github.com/philips-labs/tabia/lib/shared" 15 | ) 16 | 17 | func TestRepositoryVisibilityToJSON(t *testing.T) { 18 | assert := assert.New(t) 19 | 20 | expectedTemplate := `{"name":"%s","visibility":"%s","created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","pushed_at":"0001-01-01T00:00:00Z"} 21 | ` 22 | 23 | var result strings.Builder 24 | jsonEnc := json.NewEncoder(&result) 25 | 26 | privRepo := github.Repository{ 27 | Name: "private-repo", 28 | Visibility: shared.Private, 29 | } 30 | err := jsonEnc.Encode(privRepo) 31 | assert.NoError(err) 32 | assert.Equal(fmt.Sprintf(expectedTemplate, "private-repo", "Private"), result.String()) 33 | 34 | var unmarshalledRepo github.Repository 35 | err = json.Unmarshal([]byte(result.String()), &unmarshalledRepo) 36 | if assert.NoError(err) { 37 | assert.Equal(shared.Private, unmarshalledRepo.Visibility) 38 | } 39 | 40 | internalRepo := github.Repository{ 41 | Name: "internal-repo", 42 | Visibility: shared.Internal, 43 | } 44 | result.Reset() 45 | err = jsonEnc.Encode(internalRepo) 46 | assert.NoError(err) 47 | assert.Equal(fmt.Sprintf(expectedTemplate, "internal-repo", "Internal"), result.String()) 48 | 49 | err = json.Unmarshal([]byte(result.String()), &unmarshalledRepo) 50 | if assert.NoError(err) { 51 | assert.Equal(shared.Internal, unmarshalledRepo.Visibility) 52 | } 53 | 54 | publicRepo := github.Repository{ 55 | Name: "public-repo", 56 | Visibility: shared.Public, 57 | } 58 | result.Reset() 59 | 60 | err = jsonEnc.Encode(publicRepo) 61 | assert.NoError(err) 62 | assert.Equal(fmt.Sprintf(expectedTemplate, "public-repo", "Public"), result.String()) 63 | 64 | err = json.Unmarshal([]byte(result.String()), &unmarshalledRepo) 65 | if assert.NoError(err) { 66 | assert.Equal(shared.Public, unmarshalledRepo.Visibility) 67 | } 68 | } 69 | 70 | func TestMap(t *testing.T) { 71 | assert := assert.New(t) 72 | 73 | owner := graphql.Owner{Login: "philips-labs"} 74 | topics := graphql.RepositoryTopics{ 75 | Nodes: []graphql.RepositoryTopic{ 76 | {Topic: graphql.Topic{Name: "opensource"}, ResourcePath: "/topics/opensource"}, 77 | {Topic: graphql.Topic{Name: "golang"}, ResourcePath: "/topics/golang"}, 78 | {Topic: graphql.Topic{Name: "graphql"}, ResourcePath: "/topics/graphql"}, 79 | }, 80 | } 81 | collaborators := graphql.Collaborators{ 82 | Nodes: []graphql.Collaborator{ 83 | {Name: "Marco Franssen", Login: "marcofranssen", AvatarURL: "https://avatars3.githubusercontent.com/u/694733?u=6aeb327c48cb88ae31eb88e680b96228f53cae51&v=4"}, 84 | {Name: "John Doe", Login: "johndoe", AvatarURL: "https://avatars3.githubusercontent.com/u/694733?u=6aeb327c48cb88ae31eb88e680b96228f53cae51&v=4"}, 85 | }, 86 | } 87 | languages := graphql.Languages{ 88 | Edges: []graphql.LanguageEdge{ 89 | {Node: graphql.LanguageNode{Name: "Go", Color: "#cc0000"}, Size: 3000}, 90 | {Node: graphql.LanguageNode{Name: "JavaScript", Color: "#0000cc"}, Size: 532}, 91 | }, 92 | } 93 | graphqlRepositories := []graphql.Repository{ 94 | {Owner: owner, Name: "private-repo", Description: "I am private ", IsPrivate: true}, 95 | {Owner: owner, Name: "internal-repo", Description: "Superb inner-source stuff", IsPrivate: true}, 96 | {Owner: owner, Name: "opensource", Description: "I'm shared with the world", RepositoryTopics: topics}, 97 | {Owner: owner, Name: "secret-repo", Description: " ** secrets ** ", IsPrivate: true, Collaborators: collaborators, Languages: languages}, 98 | } 99 | 100 | privateRepos := []*gh.Repository{ 101 | {Name: stringPointer("private-repo")}, 102 | {Name: stringPointer("secret-repo")}, 103 | } 104 | ghRepos, err := github.Map(graphqlRepositories, privateRepos) 105 | if !assert.NoError(err) { 106 | return 107 | } 108 | 109 | assert.Len(ghRepos, 4) 110 | assert.Equal(shared.Private, ghRepos[0].Visibility) 111 | assert.Equal(shared.Internal, ghRepos[1].Visibility) 112 | assert.Equal(shared.Public, ghRepos[2].Visibility) 113 | assert.Equal(shared.Private, ghRepos[3].Visibility) 114 | 115 | assert.Equal(owner.Login, ghRepos[0].Owner) 116 | assert.Equal(owner.Login, ghRepos[1].Owner) 117 | assert.Equal(owner.Login, ghRepos[2].Owner) 118 | assert.Equal(owner.Login, ghRepos[3].Owner) 119 | 120 | assert.Equal("I am private", ghRepos[0].Description) 121 | assert.Equal("Superb inner-source stuff", ghRepos[1].Description) 122 | assert.Equal("I'm shared with the world", ghRepos[2].Description) 123 | assert.Equal("** secrets **", ghRepos[3].Description) 124 | 125 | assert.Equal("opensource", ghRepos[2].Topics[0].Name) 126 | assert.Equal("https://github.com/topics/opensource", ghRepos[2].Topics[0].URL) 127 | assert.Equal("golang", ghRepos[2].Topics[1].Name) 128 | assert.Equal("https://github.com/topics/golang", ghRepos[2].Topics[1].URL) 129 | assert.Equal("graphql", ghRepos[2].Topics[2].Name) 130 | assert.Equal("https://github.com/topics/graphql", ghRepos[2].Topics[2].URL) 131 | 132 | assert.Len(ghRepos[3].Collaborators, 2) 133 | assert.Equal("Marco Franssen", ghRepos[3].Collaborators[0].Name) 134 | assert.Equal("marcofranssen", ghRepos[3].Collaborators[0].Login) 135 | assert.Equal("https://avatars3.githubusercontent.com/u/694733?u=6aeb327c48cb88ae31eb88e680b96228f53cae51&v=4", ghRepos[3].Collaborators[0].AvatarURL) 136 | 137 | assert.Equal("John Doe", ghRepos[3].Collaborators[1].Name) 138 | assert.Equal("johndoe", ghRepos[3].Collaborators[1].Login) 139 | assert.Equal("https://avatars3.githubusercontent.com/u/694733?u=6aeb327c48cb88ae31eb88e680b96228f53cae51&v=4", ghRepos[3].Collaborators[1].AvatarURL) 140 | 141 | assert.Len(ghRepos[3].Languages, 2) 142 | assert.Equal("Go", ghRepos[3].Languages[0].Name) 143 | assert.Equal(3000, ghRepos[3].Languages[0].Size) 144 | assert.Equal("#cc0000", ghRepos[3].Languages[0].Color) 145 | assert.Equal("JavaScript", ghRepos[3].Languages[1].Name) 146 | assert.Equal(532, ghRepos[3].Languages[1].Size) 147 | assert.Equal("#0000cc", ghRepos[3].Languages[1].Color) 148 | } 149 | 150 | func stringPointer(s string) *string { 151 | return &s 152 | } 153 | -------------------------------------------------------------------------------- /lib/gitlab/client.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/hashicorp/go-cleanhttp" 7 | "github.com/xanzy/go-gitlab" 8 | 9 | "github.com/philips-labs/tabia/lib/transport" 10 | ) 11 | 12 | type Client struct { 13 | *gitlab.Client 14 | } 15 | 16 | func NewClientWithTokenAuth(baseUrl, token string, writer io.Writer) (*Client, error) { 17 | httpClient := cleanhttp.DefaultPooledClient() 18 | if writer != nil { 19 | httpClient.Transport = transport.TeeRoundTripper{ 20 | RoundTripper: httpClient.Transport, 21 | Writer: writer, 22 | } 23 | } 24 | c, err := gitlab.NewClient( 25 | token, 26 | gitlab.WithHTTPClient(httpClient), 27 | gitlab.WithBaseURL(baseUrl), 28 | ) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | return &Client{c}, nil 34 | } 35 | -------------------------------------------------------------------------------- /lib/gitlab/filter.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | 8 | "github.com/antonmedv/expr" 9 | 10 | "github.com/philips-labs/tabia/lib/shared" 11 | ) 12 | 13 | // RepositoryFilterEnv filter environment for repositories 14 | type RepositoryFilterEnv struct { 15 | Repositories []Repository 16 | } 17 | 18 | // Contains reports wether substring is in s. 19 | func (RepositoryFilterEnv) Contains(s, substring string) bool { 20 | return strings.Contains(s, substring) 21 | } 22 | 23 | // IsPublic indicates if a repository has public visibility. 24 | func (r Repository) IsPublic() bool { 25 | return r.Visibility == shared.Public 26 | } 27 | 28 | // IsInternal indicates if a repository has internal visibility. 29 | func (r Repository) IsInternal() bool { 30 | return r.Visibility == shared.Internal 31 | } 32 | 33 | // IsPrivate indicates if a repository has private visibility. 34 | func (r Repository) IsPrivate() bool { 35 | return r.Visibility == shared.Private 36 | } 37 | 38 | // LastActivitySince indicates if a repository has been active since the given date. 39 | // Date has to be given in RFC3339 format, e.g. `2006-01-02T15:04:05Z07:00`. 40 | func (r Repository) LastActivitySince(date string) bool { 41 | return equalOrAfter(*r.LastActivityAt, date) 42 | } 43 | 44 | // CreatedSince indicates if a repository has been created since the given date. 45 | // Date has to be given in RFC3339 format, e.g. `2006-01-02T15:04:05Z07:00`. 46 | func (r Repository) CreatedSince(date string) bool { 47 | return equalOrAfter(*r.CreatedAt, date) 48 | } 49 | 50 | func equalOrAfter(a time.Time, date string) bool { 51 | since, err := time.Parse(time.RFC3339, date) 52 | if err != nil { 53 | return true 54 | } 55 | 56 | return a.Equal(since) || a.After(since) 57 | } 58 | 59 | // Reduce filters the repositories based on the given filter 60 | func Reduce(repositories []Repository, filter string) ([]Repository, error) { 61 | if strings.TrimSpace(filter) == "" { 62 | return repositories, nil 63 | } 64 | 65 | program, err := expr.Compile(fmt.Sprintf("filter(Repositories, %s)", filter)) 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | result, err := expr.Run(program, RepositoryFilterEnv{repositories}) 71 | if err != nil { 72 | return nil, err 73 | } 74 | var repos []Repository 75 | for _, repo := range result.([]interface{}) { 76 | repos = append(repos, repo.(Repository)) 77 | } 78 | return repos, nil 79 | } 80 | 81 | // ConvertFiltersToListProjectOptions converts the filter expressions to ListProjectOptions 82 | func ConvertFiltersToListProjectOptions(filter string) []ListProjectOptionsFunc { 83 | var projectsFilters []ListProjectOptionsFunc 84 | 85 | if strings.Contains(filter, ".IsPublic()") { 86 | projectsFilters = append(projectsFilters, WithPublicVisibility) 87 | } else if strings.Contains(filter, ".IsInternal()") { 88 | projectsFilters = append(projectsFilters, WithInternalVisibility) 89 | } else if strings.Contains(filter, ".IsPrivate()") { 90 | projectsFilters = append(projectsFilters, WithPrivateVisibility) 91 | } 92 | 93 | return projectsFilters 94 | } 95 | -------------------------------------------------------------------------------- /lib/gitlab/filter_test.go: -------------------------------------------------------------------------------- 1 | package gitlab_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/philips-labs/tabia/lib/gitlab" 11 | "github.com/philips-labs/tabia/lib/shared" 12 | ) 13 | 14 | func TestReduce(t *testing.T) { 15 | assert := assert.New(t) 16 | 17 | repos := []gitlab.Repository{ 18 | gitlab.Repository{ 19 | ID: 1, 20 | Name: "cool", Visibility: shared.Public, Owner: "research", 21 | CreatedAt: timePointer(time.Now().Add(-24 * time.Hour)), 22 | LastActivityAt: timePointer(time.Now().Add(-24 * time.Hour)), 23 | }, 24 | gitlab.Repository{ 25 | ID: 2, 26 | Name: "stuff", Visibility: shared.Public, Owner: "research", 27 | CreatedAt: timePointer(time.Now().Add(-96 * time.Hour)), 28 | LastActivityAt: timePointer(time.Now().Add(-96 * time.Hour)), 29 | }, 30 | gitlab.Repository{ 31 | ID: 3, 32 | Name: "happens", Visibility: shared.Public, Owner: "research", 33 | CreatedAt: timePointer(time.Now().Add(-24 * time.Hour)), 34 | LastActivityAt: timePointer(time.Now().Add(-24 * time.Hour)), 35 | }, 36 | gitlab.Repository{ 37 | ID: 4, 38 | Name: "at", Visibility: shared.Internal, Owner: "research", 39 | CreatedAt: timePointer(time.Now().Add(-48 * time.Hour)), 40 | LastActivityAt: timePointer(time.Now().Add(-48 * time.Hour)), 41 | }, 42 | gitlab.Repository{ 43 | ID: 5, 44 | Name: "philips", Visibility: shared.Private, Owner: "research", 45 | CreatedAt: timePointer(time.Now().Add(-24 * time.Hour)), 46 | LastActivityAt: timePointer(time.Now().Add(-24 * time.Hour)), 47 | }, 48 | } 49 | 50 | reduced, err := gitlab.Reduce(repos, "") 51 | if assert.NoError(err) { 52 | assert.Len(reduced, 5) 53 | assert.ElementsMatch(reduced, repos) 54 | } 55 | 56 | reduced, err = gitlab.Reduce(repos, `.Name == "stuff"`) 57 | if assert.NoError(err) { 58 | assert.Len(reduced, 1) 59 | assert.Contains(reduced, repos[1]) 60 | } 61 | 62 | reduced, err = gitlab.Reduce(repos, `{ .Name == "stuff" }`) 63 | if assert.NoError(err) { 64 | assert.Len(reduced, 1) 65 | assert.Contains(reduced, repos[1]) 66 | } 67 | 68 | reduced, err = gitlab.Reduce(repos, `{ .IsPublic() }`) 69 | if assert.NoError(err) { 70 | assert.Len(reduced, 3) 71 | assert.Contains(reduced, repos[0]) 72 | assert.Contains(reduced, repos[1]) 73 | assert.Contains(reduced, repos[2]) 74 | } 75 | 76 | reduced, err = gitlab.Reduce(repos, `{ .IsInternal() }`) 77 | if assert.NoError(err) { 78 | assert.Len(reduced, 1) 79 | assert.Contains(reduced, repos[3]) 80 | } 81 | 82 | reduced, err = gitlab.Reduce(repos, `{ .IsPrivate() }`) 83 | if assert.NoError(err) { 84 | assert.Len(reduced, 1) 85 | assert.Contains(reduced, repos[4]) 86 | } 87 | 88 | reduced, err = gitlab.Reduce(repos, `{ Contains(.Name, "a") }`) 89 | if assert.NoError(err) { 90 | assert.Len(reduced, 2) 91 | assert.Contains(reduced, repos[2]) 92 | assert.Contains(reduced, repos[3]) 93 | } 94 | 95 | since := time.Now().Add(-25 * time.Hour).Format(time.RFC3339) 96 | reduced, err = gitlab.Reduce(repos, fmt.Sprintf(`{ .CreatedSince("%s") }`, since)) 97 | if assert.NoError(err) { 98 | assert.Len(reduced, 3) 99 | assert.Contains(reduced, repos[0]) 100 | assert.Contains(reduced, repos[2]) 101 | assert.Contains(reduced, repos[4]) 102 | } 103 | 104 | since = time.Now().Add(-97 * time.Hour).Format(time.RFC3339) 105 | reduced, err = gitlab.Reduce(repos, fmt.Sprintf(`{ .LastActivitySince("%s") }`, since)) 106 | if assert.NoError(err) { 107 | assert.Len(reduced, 5) 108 | assert.Contains(reduced, repos[0]) 109 | assert.Contains(reduced, repos[1]) 110 | assert.Contains(reduced, repos[2]) 111 | assert.Contains(reduced, repos[3]) 112 | assert.Contains(reduced, repos[4]) 113 | } 114 | } 115 | 116 | func TestReduceWrongExpression(t *testing.T) { 117 | assert := assert.New(t) 118 | 119 | repos := []gitlab.Repository{ 120 | gitlab.Repository{Name: "cool", Visibility: shared.Public}, 121 | } 122 | 123 | reduced, err := gitlab.Reduce(repos, `.Name = "cool"`) 124 | assert.Error(err) 125 | assert.EqualError(err, "unexpected token Operator(\"=\") (1:28)\n | filter(Repositories, .Name = \"cool\")\n | ...........................^") 126 | assert.Nil(reduced) 127 | 128 | reduced, err = gitlab.Reduce(repos, `{ UnExistingFunc(.URL, "stuff") }`) 129 | assert.Error(err) 130 | assert.EqualError(err, "cannot fetch UnExistingFunc from gitlab.RepositoryFilterEnv (1:24)\n | filter(Repositories, { UnExistingFunc(.URL, \"stuff\") })\n | .......................^") 131 | assert.Nil(reduced) 132 | } 133 | 134 | func timePointer(t time.Time) *time.Time { 135 | return &t 136 | } 137 | -------------------------------------------------------------------------------- /lib/gitlab/repositories.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "time" 7 | 8 | "github.com/xanzy/go-gitlab" 9 | 10 | "github.com/philips-labs/tabia/lib/shared" 11 | ) 12 | 13 | type Repository struct { 14 | ID int `json:"id,omitempty"` 15 | Name string `json:"name,omitempty"` 16 | Description string `json:"description,omitempty"` 17 | Owner string `json:"owner,omitempty"` 18 | URL string `json:"url,omitempty"` 19 | SSHURL string `json:"sshurl,omitempty"` 20 | CreatedAt *time.Time `json:"created_at,omitempty"` 21 | LastActivityAt *time.Time `json:"last_activity_at,omitempty"` 22 | Visibility shared.Visibility `json:"visibility,omitempty"` 23 | } 24 | 25 | type ListProjectOptionsFunc func(*gitlab.ListProjectsOptions) 26 | 27 | func WithPrivateVisibility(opt *gitlab.ListProjectsOptions) { 28 | vis := gitlab.PrivateVisibility 29 | opt.Visibility = &vis 30 | } 31 | 32 | func WithPublicVisibility(opt *gitlab.ListProjectsOptions) { 33 | vis := gitlab.PublicVisibility 34 | opt.Visibility = &vis 35 | } 36 | 37 | func WithInternalVisibility(opt *gitlab.ListProjectsOptions) { 38 | vis := gitlab.InternalVisibility 39 | opt.Visibility = &vis 40 | } 41 | 42 | func WithPaging(page, items int) gitlab.ListOptions { 43 | return gitlab.ListOptions{ 44 | PerPage: items, 45 | Page: page, 46 | } 47 | } 48 | 49 | func BuildListProjectsOptions(optionsFuncs ...ListProjectOptionsFunc) *gitlab.ListProjectsOptions { 50 | opt := &gitlab.ListProjectsOptions{ 51 | ListOptions: WithPaging(1, 100), 52 | } 53 | 54 | for _, optionFunc := range optionsFuncs { 55 | optionFunc(opt) 56 | } 57 | 58 | return opt 59 | } 60 | 61 | func (c *Client) ListRepositories(ctx context.Context, optionsFuncs ...ListProjectOptionsFunc) ([]Repository, error) { 62 | opt := BuildListProjectsOptions(optionsFuncs...) 63 | 64 | var repos []Repository 65 | for { 66 | projects, resp, err := c.Projects.ListProjects(opt, gitlab.WithContext(ctx)) 67 | if err != nil { 68 | return repos, err 69 | } 70 | defer resp.Body.Close() 71 | repos = append(repos, Map(projects)...) 72 | 73 | if resp.CurrentPage >= resp.TotalPages { 74 | break 75 | } 76 | 77 | opt.Page = resp.NextPage 78 | } 79 | return repos, nil 80 | } 81 | 82 | func Map(projects []*gitlab.Project) []Repository { 83 | repos := make([]Repository, len(projects)) 84 | for i, project := range projects { 85 | repos[i] = Repository{ 86 | ID: project.ID, 87 | Name: project.Name, 88 | Description: strings.TrimSpace(project.Description), 89 | Owner: mapOwner(project.Owner), 90 | URL: project.WebURL, 91 | SSHURL: project.SSHURLToRepo, 92 | CreatedAt: project.CreatedAt, 93 | LastActivityAt: project.LastActivityAt, 94 | Visibility: shared.VisibilityFromText(string(project.Visibility)), 95 | } 96 | } 97 | return repos 98 | } 99 | 100 | func mapOwner(owner *gitlab.User) string { 101 | if owner != nil { 102 | return owner.Name 103 | } 104 | return "" 105 | } 106 | -------------------------------------------------------------------------------- /lib/gitlab/repositories_test.go: -------------------------------------------------------------------------------- 1 | package gitlab_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | gl "github.com/xanzy/go-gitlab" 8 | 9 | "github.com/philips-labs/tabia/lib/gitlab" 10 | ) 11 | 12 | func TestMap(t *testing.T) { 13 | assert := assert.New(t) 14 | 15 | projects := []*gl.Project{ 16 | &gl.Project{}, 17 | } 18 | 19 | repos := gitlab.Map(projects) 20 | 21 | assert.Len(repos, len(projects)) 22 | } 23 | 24 | func TestBuildListProjectsOptions(t *testing.T) { 25 | assert := assert.New(t) 26 | 27 | opt := gitlab.BuildListProjectsOptions() 28 | assert.Equal(gl.ListOptions{ 29 | PerPage: 100, 30 | Page: 1, 31 | }, opt.ListOptions) 32 | assert.Equal((*gl.VisibilityValue)(nil), opt.Visibility) 33 | 34 | opt = gitlab.BuildListProjectsOptions(gitlab.WithPublicVisibility) 35 | assert.Equal(gl.ListOptions{ 36 | PerPage: 100, 37 | Page: 1, 38 | }, opt.ListOptions) 39 | assert.Equal(visibilityValuePtr(gl.PublicVisibility), opt.Visibility) 40 | 41 | opt = gitlab.BuildListProjectsOptions(gitlab.WithInternalVisibility) 42 | assert.Equal(gl.ListOptions{ 43 | PerPage: 100, 44 | Page: 1, 45 | }, opt.ListOptions) 46 | assert.Equal(visibilityValuePtr(gl.InternalVisibility), opt.Visibility) 47 | 48 | opt = gitlab.BuildListProjectsOptions(gitlab.WithPrivateVisibility) 49 | assert.Equal(gl.ListOptions{ 50 | PerPage: 100, 51 | Page: 1, 52 | }, opt.ListOptions) 53 | assert.Equal(visibilityValuePtr(gl.PrivateVisibility), opt.Visibility) 54 | } 55 | 56 | func visibilityValuePtr(v gl.VisibilityValue) *gl.VisibilityValue { 57 | return &v 58 | } 59 | -------------------------------------------------------------------------------- /lib/grimoirelab/bitbucket.go: -------------------------------------------------------------------------------- 1 | package grimoirelab 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "os" 7 | 8 | "github.com/philips-labs/tabia/lib/bitbucket" 9 | ) 10 | 11 | // BitbucketMetadataFactory allows to provide a custom generated metadata 12 | type BitbucketMetadataFactory func(repo bitbucket.Repository) Metadata 13 | 14 | // ConvertBitbucketToProjectsJSON converts the repositories into grimoirelab projects.json 15 | func ConvertBitbucketToProjectsJSON(repos []bitbucket.Repository, metadataFactory BitbucketMetadataFactory) Projects { 16 | results := make(Projects) 17 | bbUser := os.Getenv("TABIA_BITBUCKET_USER") 18 | bbToken := os.Getenv("TABIA_BITBUCKET_TOKEN") 19 | basicAuth := fmt.Sprintf("%s:%s", bbUser, bbToken) 20 | for _, repo := range repos { 21 | project, found := results[repo.Project.Name] 22 | if !found { 23 | results[repo.Project.Name] = &Project{} 24 | project = results[repo.Project.Name] 25 | project.Git = make([]string, 0) 26 | } 27 | updateFromBitbucketProject(project, repo, basicAuth, metadataFactory) 28 | } 29 | 30 | return results 31 | } 32 | 33 | func updateFromBitbucketProject(project *Project, repo bitbucket.Repository, basicAuth string, metadataFactory BitbucketMetadataFactory) { 34 | project.Metadata = metadataFactory(repo) 35 | link := getCloneLink(repo, "http") 36 | if link != "" { 37 | if !repo.Public { 38 | u, _ := url.Parse(link) 39 | link = fmt.Sprintf("%s://%s@%s%s", u.Scheme, basicAuth, u.Hostname(), u.EscapedPath()) 40 | } 41 | project.Git = append(project.Git, link) 42 | } 43 | } 44 | 45 | func getCloneLink(repo bitbucket.Repository, linkName string) string { 46 | for _, l := range repo.Links.Clone { 47 | if l.Name == linkName { 48 | return l.Href 49 | } 50 | } 51 | return "" 52 | } 53 | -------------------------------------------------------------------------------- /lib/grimoirelab/bitbucket_test.go: -------------------------------------------------------------------------------- 1 | package grimoirelab_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | 9 | "github.com/philips-labs/tabia/lib/bitbucket" 10 | "github.com/philips-labs/tabia/lib/grimoirelab" 11 | ) 12 | 13 | func TestConvertBitbucketProjectsJSON(t *testing.T) { 14 | if len(os.Getenv("TABIA_BITBUCKET_USER")) == 0 || len(os.Getenv("TABIA_BITBUCKET_TOKEN")) == 0 { 15 | t.Skip("skipping integration test, depending on environment variable") 16 | } 17 | assert := assert.New(t) 18 | 19 | bbUser := os.Getenv("TABIA_BITBUCKET_USER") 20 | bbToken := os.Getenv("TABIA_BITBUCKET_TOKEN") 21 | 22 | repos := []bitbucket.Repository{ 23 | { 24 | Project: bitbucket.Project{Name: "P1"}, 25 | Name: "R1", 26 | Public: true, 27 | Links: bitbucket.Links{ 28 | Clone: []bitbucket.CloneLink{ 29 | {Name: "http", Href: "https://bitbucket.org/scm/p1/r1.git"}, 30 | }, 31 | }, 32 | }, 33 | { 34 | Project: bitbucket.Project{Name: "P1"}, 35 | Name: "R2", 36 | Public: false, 37 | Links: bitbucket.Links{ 38 | Clone: []bitbucket.CloneLink{ 39 | {Name: "http", Href: "https://bitbucket.org/scm/p1/r2.git"}, 40 | }, 41 | }, 42 | }, 43 | { 44 | Project: bitbucket.Project{Name: "P2"}, 45 | Name: "R1", 46 | Public: false, 47 | Links: bitbucket.Links{ 48 | Clone: []bitbucket.CloneLink{ 49 | {Name: "http", Href: "https://bitbucket.org/scm/p2/r1.git"}, 50 | }, 51 | }, 52 | }, 53 | { 54 | Project: bitbucket.Project{Name: "P2"}, 55 | Name: "R2", 56 | Public: false, 57 | Links: bitbucket.Links{ 58 | Clone: []bitbucket.CloneLink{ 59 | {Name: "http", Href: "https://bitbucket.org/scm/p2/r2.git"}, 60 | }, 61 | }, 62 | }, 63 | { 64 | Project: bitbucket.Project{Name: "P2"}, 65 | Name: "R3", 66 | Public: true, 67 | Links: bitbucket.Links{ 68 | Clone: []bitbucket.CloneLink{ 69 | {Name: "http", Href: "https://bitbucket.org/scm/p2/r3.git"}, 70 | }, 71 | }, 72 | }, 73 | } 74 | 75 | projects := grimoirelab.ConvertBitbucketToProjectsJSON(repos, func(repo bitbucket.Repository) grimoirelab.Metadata { 76 | return grimoirelab.Metadata{ 77 | "title": repo.Project.Name, 78 | "program": "One Codebase", 79 | } 80 | }) 81 | 82 | if assert.Len(projects, 2) { 83 | gitP1 := projects["P1"].Git 84 | if assert.Len(gitP1, 2) { 85 | assert.Equal(gitP1[0], "https://bitbucket.org/scm/p1/r1.git") 86 | assertUrlHasBasicAuth(t, gitP1[1], "https", bbUser, bbToken, "bitbucket.org", "/scm/p1/r2.git") 87 | } 88 | assert.Len(projects["P1"].Metadata, 2) 89 | 90 | gitP2 := projects["P2"].Git 91 | if assert.Len(gitP2, 3) { 92 | assertUrlHasBasicAuth(t, gitP2[0], "https", bbUser, bbToken, "bitbucket.org", "/scm/p2/r1.git") 93 | assertUrlHasBasicAuth(t, gitP2[1], "https", bbUser, bbToken, "bitbucket.org", "/scm/p2/r2.git") 94 | assert.Equal(gitP2[2], "https://bitbucket.org/scm/p2/r3.git") 95 | } 96 | assert.Len(projects["P2"].Metadata, 2) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /lib/grimoirelab/github.go: -------------------------------------------------------------------------------- 1 | package grimoirelab 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/url" 8 | "os" 9 | "regexp" 10 | 11 | "github.com/philips-labs/tabia/lib/github" 12 | "github.com/philips-labs/tabia/lib/shared" 13 | ) 14 | 15 | // GithubMetadataFactory allows to provide a custom generated metadata 16 | type GithubMetadataFactory func(repo github.Repository) Metadata 17 | 18 | // GithubProjectMatcher matches a repository with a project 19 | type GithubProjectMatcher struct { 20 | Rules map[string]GithubProjectMatcherRule `json:"rules,omitempty"` 21 | } 22 | 23 | // GithubProjectMatcherRule rule that matches a repository to a project 24 | type GithubProjectMatcherRule struct { 25 | URL *Regexp `json:"url,omitempty"` 26 | } 27 | 28 | // Regexp embeds a regexp.Regexp, and adds Text/JSON 29 | // (un)marshaling. 30 | type Regexp struct { 31 | regexp.Regexp 32 | } 33 | 34 | // Compile wraps the result of the standard library's 35 | // regexp.Compile, for easy (un)marshaling. 36 | func Compile(expr string) (*Regexp, error) { 37 | r, err := regexp.Compile(expr) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | return &Regexp{*r}, nil 43 | } 44 | 45 | // MustCompile wraps the result of the standard library's 46 | // regexp.Compile, for easy (un)marshaling. 47 | func MustCompile(expr string) *Regexp { 48 | r := regexp.MustCompile(expr) 49 | return &Regexp{*r} 50 | } 51 | 52 | // UnmarshalText satisfies the encoding.TextMarshaler interface, 53 | // also used by json.Unmarshal. 54 | func (r *Regexp) UnmarshalText(b []byte) error { 55 | rr, err := Compile(string(b)) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | *r = *rr 61 | 62 | return nil 63 | } 64 | 65 | // MarshalText satisfies the encoding.TextMarshaler interface, 66 | // also used by json.Marshal. 67 | func (r *Regexp) MarshalText() ([]byte, error) { 68 | return []byte(r.String()), nil 69 | } 70 | 71 | // NewGithubProjectMatcherFromJSON initializes GithubProjectMatcher from json 72 | func NewGithubProjectMatcherFromJSON(data io.Reader) (*GithubProjectMatcher, error) { 73 | var m GithubProjectMatcher 74 | err := json.NewDecoder(data).Decode(&m) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | return &m, err 80 | } 81 | 82 | // ConvertGithubToProjectsJSON converts the repositories into grimoirelab projects.json 83 | func ConvertGithubToProjectsJSON(repos []github.Repository, metadataFactory GithubMetadataFactory, projectMatcher *GithubProjectMatcher) Projects { 84 | results := make(Projects) 85 | bbUser := os.Getenv("TABIA_GITHUB_USER") 86 | bbToken := os.Getenv("TABIA_GITHUB_TOKEN") 87 | basicAuth := fmt.Sprintf("%s:%s", bbUser, bbToken) 88 | for _, repo := range repos { 89 | projectName := getProjectName(repo, projectMatcher) 90 | project, found := results[projectName] 91 | if !found { 92 | results[projectName] = &Project{} 93 | project = results[projectName] 94 | project.Git = make([]string, 0) 95 | } 96 | updateFromGithubProject(project, repo, basicAuth, metadataFactory) 97 | } 98 | 99 | return results 100 | } 101 | 102 | func getProjectName(repo github.Repository, projectMatcher *GithubProjectMatcher) string { 103 | if projectMatcher != nil { 104 | for k, v := range projectMatcher.Rules { 105 | if v.URL != nil && v.URL.MatchString(repo.URL) { 106 | return k 107 | } 108 | } 109 | } 110 | 111 | // fallback to github organization name 112 | return repo.Owner 113 | } 114 | 115 | func updateFromGithubProject(project *Project, repo github.Repository, basicAuth string, metadataFactory GithubMetadataFactory) { 116 | project.Metadata = metadataFactory(repo) 117 | link := repo.URL 118 | if link != "" { 119 | if repo.Visibility != shared.Public { 120 | u, _ := url.Parse(link) 121 | link = fmt.Sprintf("%s://%s@%s%s", u.Scheme, basicAuth, u.Hostname(), u.EscapedPath()) 122 | } 123 | project.Git = append(project.Git, link+".git") 124 | project.Github = append(project.Github, link) 125 | project.GithubRepo = append(project.GithubRepo, link) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /lib/grimoirelab/github_test.go: -------------------------------------------------------------------------------- 1 | package grimoirelab_test 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/philips-labs/tabia/lib/github" 11 | "github.com/philips-labs/tabia/lib/grimoirelab" 12 | "github.com/philips-labs/tabia/lib/shared" 13 | ) 14 | 15 | func TestNewGithubProjectMatcherFromJSON(t *testing.T) { 16 | assert := assert.New(t) 17 | 18 | json := strings.NewReader(`{ 19 | "rules": { 20 | "My Project": { "url": "(?i)foo|Bar|BAZ" } 21 | } 22 | }`) 23 | m, err := grimoirelab.NewGithubProjectMatcherFromJSON(json) 24 | if assert.NoError(err) { 25 | assert.Equal("(?i)foo|Bar|BAZ", m.Rules["My Project"].URL.String()) 26 | } 27 | 28 | json = strings.NewReader(`{ 29 | "rules": { 30 | "My Project": { "url": "" }, 31 | } 32 | }`) 33 | m, err = grimoirelab.NewGithubProjectMatcherFromJSON(json) 34 | assert.EqualError(err, "invalid character '}' looking for beginning of object key string") 35 | assert.Nil(m) 36 | 37 | json = strings.NewReader(`{ 38 | "rules": { 39 | "My Project": { "url": "(invalid|regex" } 40 | } 41 | }`) 42 | m, err = grimoirelab.NewGithubProjectMatcherFromJSON(json) 43 | assert.EqualError(err, "error parsing regexp: missing closing ): `(invalid|regex`") 44 | assert.Nil(m) 45 | } 46 | 47 | func TestConvertGithubProjectsJSON(t *testing.T) { 48 | if len(os.Getenv("TABIA_GITHUB_USER")) == 0 || len(os.Getenv("TABIA_GITHUB_TOKEN")) == 0 { 49 | t.Skip("skipping integration test, depending on environment variable") 50 | } 51 | assert := assert.New(t) 52 | 53 | ghUser := os.Getenv("TABIA_GITHUB_USER") 54 | ghToken := os.Getenv("TABIA_GITHUB_TOKEN") 55 | 56 | repos := []github.Repository{ 57 | { 58 | Name: "R1", 59 | Visibility: shared.Public, 60 | URL: "https://github.com/philips-software/logproxy", 61 | Owner: "philips-software", 62 | }, 63 | { 64 | Name: "R1", 65 | Visibility: shared.Private, 66 | URL: "https://github.com/philips-labs/tabia", 67 | Owner: "philips-labs", 68 | }, 69 | } 70 | 71 | projects := grimoirelab.ConvertGithubToProjectsJSON( 72 | repos, 73 | func(repo github.Repository) grimoirelab.Metadata { 74 | return grimoirelab.Metadata{ 75 | "title": repo.Owner, 76 | "program": "One Codebase", 77 | } 78 | }, 79 | &grimoirelab.GithubProjectMatcher{ 80 | Rules: map[string]grimoirelab.GithubProjectMatcherRule{ 81 | "One Codebase": {URL: grimoirelab.MustCompile("(?i)Tabia")}, 82 | }, 83 | }, 84 | ) 85 | 86 | if assert.Len(projects, 2) { 87 | if assert.Len(projects["philips-software"].Git, 1) { 88 | assert.Equal("https://github.com/philips-software/logproxy.git", projects["philips-software"].Git[0]) 89 | assert.Equal("https://github.com/philips-software/logproxy", projects["philips-software"].Github[0]) 90 | assert.Equal("https://github.com/philips-software/logproxy", projects["philips-software"].GithubRepo[0]) 91 | } 92 | assert.Len(projects["philips-software"].Metadata, 2) 93 | 94 | if assert.Len(projects["One Codebase"].Git, 1) { 95 | assertUrlHasBasicAuth(t, projects["One Codebase"].Git[0], "https", ghUser, ghToken, "github.com", "/philips-labs/tabia.git") 96 | } 97 | if assert.Len(projects["One Codebase"].Github, 1) { 98 | assertUrlHasBasicAuth(t, projects["One Codebase"].Github[0], "https", ghUser, ghToken, "github.com", "/philips-labs/tabia") 99 | } 100 | if assert.Len(projects["One Codebase"].GithubRepo, 1) { 101 | assertUrlHasBasicAuth(t, projects["One Codebase"].GithubRepo[0], "https", ghUser, ghToken, "github.com", "/philips-labs/tabia") 102 | } 103 | assert.Len(projects["One Codebase"].Metadata, 2) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /lib/grimoirelab/grimoirelab.go: -------------------------------------------------------------------------------- 1 | package grimoirelab 2 | 3 | // Projects holds all projects to be loaded in Grimoirelab 4 | type Projects map[string]*Project 5 | 6 | // Project holds the project resources and metadata 7 | type Project struct { 8 | Metadata Metadata `json:"meta,omitempty"` 9 | Git []string `json:"git,omitempty"` 10 | Github []string `json:"github,omitempty"` 11 | GithubRepo []string `json:"github:repo,omitempty"` 12 | } 13 | 14 | // Metadata hold metadata for a given project 15 | type Metadata map[string]string 16 | -------------------------------------------------------------------------------- /lib/grimoirelab/grimoirelab_test.go: -------------------------------------------------------------------------------- 1 | package grimoirelab_test 2 | 3 | import ( 4 | "net/url" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func assertUrlHasBasicAuth(t *testing.T, uri, scheme, user, password, hostname, path string) { 11 | assert := assert.New(t) 12 | u, err := url.Parse(uri) 13 | assert.NoError(err) 14 | assert.Equal(scheme, u.Scheme) 15 | assert.Equal(user, u.User.Username()) 16 | pass, isSet := u.User.Password() 17 | assert.True(isSet) 18 | assert.Equal(password, pass) 19 | assert.Equal(hostname, u.Hostname()) 20 | assert.Equal(path, u.EscapedPath()) 21 | } 22 | -------------------------------------------------------------------------------- /lib/output/output.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "text/template" 7 | ) 8 | 9 | // PrintJSON prints the json using indentation using the given writer 10 | func PrintJSON(w io.Writer, data interface{}) error { 11 | enc := json.NewEncoder(w) 12 | enc.SetEscapeHTML(false) 13 | enc.SetIndent("", " ") 14 | 15 | return enc.Encode(data) 16 | } 17 | 18 | // PrintUsingTemplate prints the data using the given template 19 | func PrintUsingTemplate(w io.Writer, templateContent []byte, data interface{}) error { 20 | tmpl, err := template.New("template").Parse(string(templateContent)) 21 | if err != nil { 22 | return err 23 | } 24 | return tmpl.Execute(w, data) 25 | } 26 | -------------------------------------------------------------------------------- /lib/output/output_test.go: -------------------------------------------------------------------------------- 1 | package output_test 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/philips-labs/tabia/lib/output" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestPrintJson(t *testing.T) { 13 | assert := assert.New(t) 14 | 15 | data := struct { 16 | Title string `json:"title"` 17 | Description string `json:"description"` 18 | Priority int `json:"prio"` 19 | }{Title: "JSON printing", Description: "Prints JSON to io.Writer", Priority: 1} 20 | 21 | var builder strings.Builder 22 | err := output.PrintJSON(&builder, data) 23 | 24 | if assert.NoError(err) { 25 | assert.Equal("{\n \"title\": \"JSON printing\",\n \"description\": \"Prints JSON to io.Writer\",\n \"prio\": 1\n}\n", builder.String()) 26 | } 27 | } 28 | 29 | func TestPrintUsingTemplate(t *testing.T) { 30 | assert := assert.New(t) 31 | 32 | data := struct { 33 | Title string `json:"title"` 34 | Description string `json:"description"` 35 | Priority int `json:"prio"` 36 | }{Title: "Markdown template", Description: "Prints markdown using template to io.Writer", Priority: 1} 37 | 38 | var builder strings.Builder 39 | err := output.PrintUsingTemplate(&builder, []byte(`# Markdown 40 | 41 | ## {{ .Title}} - {{ .Priority }} 42 | 43 | {{ .Description }} 44 | `), data) 45 | 46 | if assert.NoError(err) { 47 | assert.Equal(`# Markdown 48 | 49 | ## Markdown template - 1 50 | 51 | Prints markdown using template to io.Writer 52 | `, builder.String()) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /lib/shared/visibility.go: -------------------------------------------------------------------------------- 1 | package shared 2 | 3 | import "strings" 4 | 5 | //go:generate stringer -type=Visibility 6 | 7 | // Visibility indicates repository visibility 8 | type Visibility int 9 | 10 | const ( 11 | // Public repositories are publicly visible 12 | Public Visibility = iota 13 | // Internal repositories are only visible to organization members 14 | Internal 15 | // Private repositories are only visible to authorized users 16 | Private 17 | ) 18 | 19 | func (v *Visibility) UnmarshalText(text []byte) error { 20 | *v = VisibilityFromText(string(text)) 21 | return nil 22 | } 23 | 24 | func (v Visibility) MarshalText() ([]byte, error) { 25 | return []byte(v.String()), nil 26 | } 27 | 28 | func VisibilityFromText(text string) Visibility { 29 | switch strings.ToLower(text) { 30 | default: 31 | return Public 32 | case "public": 33 | return Public 34 | case "internal": 35 | return Internal 36 | case "private": 37 | return Private 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /lib/shared/visibility_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=Visibility"; DO NOT EDIT. 2 | 3 | package shared 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[Public-0] 12 | _ = x[Internal-1] 13 | _ = x[Private-2] 14 | } 15 | 16 | const _Visibility_name = "PublicInternalPrivate" 17 | 18 | var _Visibility_index = [...]uint8{0, 6, 14, 21} 19 | 20 | func (i Visibility) String() string { 21 | if i < 0 || i >= Visibility(len(_Visibility_index)-1) { 22 | return "Visibility(" + strconv.FormatInt(int64(i), 10) + ")" 23 | } 24 | return _Visibility_name[_Visibility_index[i]:_Visibility_index[i+1]] 25 | } 26 | -------------------------------------------------------------------------------- /lib/shared/visibility_test.go: -------------------------------------------------------------------------------- 1 | package shared_test 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | 9 | "github.com/philips-labs/tabia/lib/shared" 10 | ) 11 | 12 | func TestRepositoryVisibility(t *testing.T) { 13 | assert := assert.New(t) 14 | 15 | assert.Equal("Public", shared.Public.String()) 16 | assert.Equal("Internal", shared.Internal.String()) 17 | assert.Equal("Private", shared.Private.String()) 18 | } 19 | 20 | func TestVisibilityFromText(t *testing.T) { 21 | assert := assert.New(t) 22 | 23 | assert.Equal(shared.Public, shared.VisibilityFromText("")) 24 | assert.Equal(shared.Public, shared.VisibilityFromText("unknown")) 25 | assert.Equal(shared.Public, shared.VisibilityFromText("public")) 26 | assert.Equal(shared.Public, shared.VisibilityFromText("PUBLIC")) 27 | assert.Equal(shared.Public, shared.VisibilityFromText("Public")) 28 | assert.Equal(shared.Internal, shared.VisibilityFromText("internal")) 29 | assert.Equal(shared.Internal, shared.VisibilityFromText("INTERNAL")) 30 | assert.Equal(shared.Internal, shared.VisibilityFromText("Internal")) 31 | assert.Equal(shared.Private, shared.VisibilityFromText("private")) 32 | assert.Equal(shared.Private, shared.VisibilityFromText("PRIVATE")) 33 | assert.Equal(shared.Private, shared.VisibilityFromText("Private")) 34 | } 35 | 36 | func TestVisibilityMarshalling(t *testing.T) { 37 | assert := assert.New(t) 38 | blob := `["Public","Internal","Private"]` 39 | 40 | var visibilities []shared.Visibility 41 | 42 | err := json.Unmarshal([]byte(blob), &visibilities) 43 | if assert.NoError(err) { 44 | newBlob, err := json.Marshal(visibilities) 45 | if assert.NoError(err) { 46 | assert.Equal(blob, string(newBlob)) 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /lib/transport/transport.go: -------------------------------------------------------------------------------- 1 | package transport 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | ) 8 | 9 | // TeeRoundTripper copies request bodies to stdout. 10 | type TeeRoundTripper struct { 11 | http.RoundTripper 12 | Writer io.Writer 13 | } 14 | 15 | // RoundTrip executes a single HTTP transaction, returning 16 | // a Response for the provided Request. 17 | func (t TeeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 18 | fmt.Fprintf(t.Writer, "%s: %s ", req.Method, req.URL) 19 | if req.Body != nil { 20 | req.Body = struct { 21 | io.Reader 22 | io.Closer 23 | }{ 24 | Reader: io.TeeReader(req.Body, t.Writer), 25 | Closer: req.Body, 26 | } 27 | } 28 | 29 | return t.RoundTripper.RoundTrip(req) 30 | } 31 | -------------------------------------------------------------------------------- /lib/transport/transport_test.go: -------------------------------------------------------------------------------- 1 | package transport_test 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | 12 | "github.com/philips-labs/tabia/lib/transport" 13 | ) 14 | 15 | func TestTeeRoundTripper(t *testing.T) { 16 | assert := assert.New(t) 17 | 18 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 19 | fmt.Fprintln(w, "Hello world") 20 | })) 21 | defer ts.Close() 22 | 23 | var writer strings.Builder 24 | client := http.Client{ 25 | Transport: transport.TeeRoundTripper{ 26 | RoundTripper: http.DefaultTransport, 27 | Writer: &writer, 28 | }, 29 | } 30 | 31 | jsonString := `{ "say": "hello-world", "to": "marco" }` 32 | json := strings.NewReader(jsonString) 33 | _, err := client.Post(ts.URL, "application/json", json) 34 | 35 | assert.NoError(err) 36 | assert.NotEmpty(writer.String()) 37 | assert.Equal(fmt.Sprintf("POST: %s %s", ts.URL, jsonString), writer.String()) 38 | 39 | writer.Reset() 40 | _, err = client.Get(ts.URL) 41 | 42 | assert.NoError(err) 43 | assert.NotEmpty(writer.String()) 44 | assert.Equal(fmt.Sprintf("GET: %s ", ts.URL), writer.String()) 45 | 46 | } 47 | -------------------------------------------------------------------------------- /tools.go: -------------------------------------------------------------------------------- 1 | // +build tools 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/goreleaser/goreleaser" 7 | _ "golang.org/x/tools/cmd/stringer" 8 | ) 9 | --------------------------------------------------------------------------------