├── files ├── file0.text ├── file0.yaml ├── file1.yaml ├── file2.yaml ├── file0.json └── file1.json ├── .vscode └── settings.json ├── apis ├── bitbucketapi │ ├── methods.go │ ├── datastructures.go │ └── apis.go ├── datastructures.go ├── provider.go ├── githubapi │ ├── method_test.go │ ├── methods.go │ ├── apis.go │ ├── datastructures.go │ └── mockapis.go ├── azureapi │ ├── methods_test.go │ ├── methods.go │ ├── datastructures.go │ ├── mockapis.go │ └── apis.go ├── gitlabapi │ ├── methods.go │ ├── datastructures.go │ └── apis.go ├── urlcomposer.go ├── urlcomposer_test.go └── http.go ├── go.mod ├── gitlabparser └── v1 │ ├── datastructures.go │ ├── commit.go │ ├── download.go │ ├── tree.go │ ├── parser.go │ └── parser_test.go ├── bitbucketparser └── v1 │ ├── datastructures.go │ ├── download.go │ ├── tree.go │ ├── commit_test.go │ ├── commit.go │ ├── parser.go │ └── parser_test.go ├── githubparser └── v1 │ ├── datastructures.go │ ├── commit_test.go │ ├── tree_test.go │ ├── commit.go │ ├── download.go │ ├── tree.go │ ├── parser.go │ └── parser_test.go ├── azureparser └── v1 │ ├── datastructures.go │ ├── commit_test.go │ ├── tree_test.go │ ├── commit.go │ ├── download.go │ ├── tree.go │ ├── parser.go │ └── parser_test.go ├── .github └── workflows │ ├── pr.yaml │ └── release.yaml ├── go.sum ├── interface.go ├── init.go ├── README.md ├── init_test.go └── LICENSE /files/file0.text: -------------------------------------------------------------------------------- 1 | name=file0 -------------------------------------------------------------------------------- /files/file0.yaml: -------------------------------------------------------------------------------- 1 | name: file0 -------------------------------------------------------------------------------- /files/file1.yaml: -------------------------------------------------------------------------------- 1 | name: file1 -------------------------------------------------------------------------------- /files/file2.yaml: -------------------------------------------------------------------------------- 1 | name: file2 -------------------------------------------------------------------------------- /files/file0.json: -------------------------------------------------------------------------------- 1 | {"name": "file0"} -------------------------------------------------------------------------------- /files/file1.json: -------------------------------------------------------------------------------- 1 | {"name": "file1"} -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "WhiteSource Advise.Diff.BaseBranch": "master" 3 | } -------------------------------------------------------------------------------- /apis/bitbucketapi/methods.go: -------------------------------------------------------------------------------- 1 | package bitbucketapi 2 | 3 | import "fmt" 4 | 5 | // ToMap convert headers to map[string]string 6 | func (h *Headers) ToMap() map[string]string { 7 | m := make(map[string]string) 8 | if h.Token != "" { 9 | m["Authorization"] = fmt.Sprintf("Bearer %s", h.Token) 10 | } 11 | return m 12 | } 13 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kubescape/go-git-url 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/chainguard-dev/git-urls v1.0.2 7 | github.com/stretchr/testify v1.3.0 8 | k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 9 | ) 10 | 11 | require ( 12 | github.com/davecgh/go-spew v1.1.1 // indirect 13 | github.com/pmezard/go-difflib v1.0.0 // indirect 14 | ) 15 | -------------------------------------------------------------------------------- /gitlabparser/v1/datastructures.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import "github.com/kubescape/go-git-url/apis/gitlabapi" 4 | 5 | type GitLabURL struct { 6 | host string 7 | owner string // repo owner 8 | repo string // repo name 9 | project string 10 | branch string 11 | path string 12 | token string // github token 13 | isFile bool 14 | 15 | gitLabAPI gitlabapi.IGitLabAPI 16 | } 17 | -------------------------------------------------------------------------------- /bitbucketparser/v1/datastructures.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "github.com/kubescape/go-git-url/apis/bitbucketapi" 5 | ) 6 | 7 | type BitBucketURL struct { 8 | host string 9 | owner string // repo owner 10 | repo string // repo name 11 | project string 12 | branch string 13 | path string 14 | token string // github token 15 | isFile bool 16 | 17 | bitBucketAPI bitbucketapi.IBitBucketAPI 18 | } 19 | -------------------------------------------------------------------------------- /githubparser/v1/datastructures.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import "github.com/kubescape/go-git-url/apis/githubapi" 4 | 5 | type GitHubURL struct { 6 | host string // default is github 7 | owner string // repo owner 8 | repo string // repo name 9 | branch string 10 | path string 11 | token string // github token 12 | isFile bool // is the URL is pointing to a file or not 13 | 14 | gitHubAPI githubapi.IGitHubAPI 15 | } 16 | -------------------------------------------------------------------------------- /azureparser/v1/datastructures.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import "github.com/kubescape/go-git-url/apis/azureapi" 4 | 5 | type AzureURL struct { 6 | host string 7 | owner string // repo owner 8 | repo string // repo name 9 | project string 10 | branch string 11 | tag string 12 | path string 13 | token string // azure token 14 | isFile bool // is the URL is pointing to a file or not 15 | // username string 16 | // password string 17 | 18 | azureAPI azureapi.IAzureAPI 19 | } 20 | -------------------------------------------------------------------------------- /bitbucketparser/v1/download.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import "fmt" 4 | 5 | func (gl *BitBucketURL) DownloadAllFiles() (map[string][]byte, map[string]error) { 6 | //TODO implement me 7 | return nil, map[string]error{"": fmt.Errorf("DownloadAllFiles is not supported")} 8 | } 9 | 10 | func (gl *BitBucketURL) DownloadFilesWithExtension(extensions []string) (map[string][]byte, map[string]error) { 11 | //TODO implement me 12 | return nil, map[string]error{"": fmt.Errorf("DownloadFilesWithExtension is not supported")} 13 | } 14 | -------------------------------------------------------------------------------- /azureparser/v1/commit_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestGetLatestCommit(t *testing.T) { 10 | { 11 | az, err := NewAzureParserWithURL(urlA) 12 | assert.NoError(t, err) 13 | latestCommit, err := az.GetLatestCommit() 14 | assert.NoError(t, err) 15 | assert.Equal(t, "4502b9b51ee3ac1ea649bacfa0f48ebdeab05f4a", latestCommit.SHA) 16 | assert.Equal(t, "GitHub", latestCommit.Committer.Name) 17 | assert.Equal(t, "", latestCommit.Committer.Email) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /apis/datastructures.go: -------------------------------------------------------------------------------- 1 | package apis 2 | 3 | import "time" 4 | 5 | type Commit struct { 6 | SHA string `json:"sha"` 7 | Author Committer `json:"author"` 8 | Committer Committer `json:"committer"` 9 | Message string `json:"message"` 10 | Files []Files `json:"files"` 11 | } 12 | 13 | type Files struct { 14 | FileSHA string `json:"sha"` 15 | Filename string `json:"filename"` 16 | } 17 | 18 | type Committer struct { 19 | Name string `json:"name"` 20 | Email string `json:"email"` 21 | Date time.Time `json:"date"` 22 | } 23 | -------------------------------------------------------------------------------- /githubparser/v1/commit_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestGetLatestCommit(t *testing.T) { 10 | { 11 | gh, err := NewGitHubParserWithURLMock(urlA) 12 | assert.NoError(t, err) 13 | latestCommit, err := gh.GetLatestCommit() 14 | assert.NoError(t, err) 15 | assert.Equal(t, "e7d287e491b4002bc59d67ad7423d8119fc89e6c", latestCommit.SHA) 16 | assert.Equal(t, "David Wertenteil", latestCommit.Committer.Name) 17 | assert.Equal(t, "dwertent@armosec.io", latestCommit.Committer.Email) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /apis/provider.go: -------------------------------------------------------------------------------- 1 | package apis 2 | 3 | import "errors" 4 | 5 | type ProviderType string 6 | 7 | const ( 8 | ProviderGitHub ProviderType = "github" 9 | ProviderAzure ProviderType = "azure" 10 | ProviderBitBucket ProviderType = "bitbucket" 11 | ProviderGitLab ProviderType = "gitlab" 12 | ) 13 | 14 | func (pt ProviderType) IsSupported() error { 15 | switch pt { 16 | case ProviderGitHub, ProviderAzure, ProviderBitBucket, ProviderGitLab: 17 | return nil 18 | } 19 | return errors.New("unsupported provider") 20 | } 21 | 22 | func (pt ProviderType) String() string { 23 | return string(pt) 24 | } 25 | -------------------------------------------------------------------------------- /azureparser/v1/tree_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestListAllNames(t *testing.T) { 10 | { 11 | az, err := NewAzureParserWithURL(urlA) 12 | assert.NoError(t, err) 13 | tree, err := az.ListAllNames() 14 | assert.NoError(t, err) 15 | assert.Equal(t, 1440, len(tree)) 16 | } 17 | } 18 | 19 | func TestGetFileExtension(t *testing.T) { 20 | assert.Equal(t, "yaml", getFileExtension("my/name.yaml")) 21 | assert.Equal(t, "txt", getFileExtension("/my/name.txt")) 22 | assert.Equal(t, "json", getFileExtension("myName.json")) 23 | assert.Equal(t, "", getFileExtension("myName")) 24 | } 25 | -------------------------------------------------------------------------------- /apis/githubapi/method_test.go: -------------------------------------------------------------------------------- 1 | package githubapi 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestListAll(t *testing.T) { 10 | tree, _ := NewMockGitHubAPI().GetRepoTree("kubescape", "go-git-url", "", nil) 11 | assert.Equal(t, 25, len(tree.ListAll())) 12 | } 13 | 14 | func TestListAllFiles(t *testing.T) { 15 | tree, _ := NewMockGitHubAPI().GetRepoTree("kubescape", "go-git-url", "", nil) 16 | assert.Equal(t, 19, len(tree.ListAllFiles())) 17 | } 18 | 19 | func TestListAllDirs(t *testing.T) { 20 | tree, _ := NewMockGitHubAPI().GetRepoTree("kubescape", "go-git-url", "", nil) 21 | assert.Equal(t, 6, len(tree.ListAllDirs())) 22 | } 23 | -------------------------------------------------------------------------------- /.github/workflows/pr.yaml: -------------------------------------------------------------------------------- 1 | name: pr-checks 2 | 3 | on: 4 | pull_request: 5 | types: [ edited, opened, synchronize, reopened ] 6 | paths-ignore: 7 | # Do not run the pipeline if only Markdown files changed 8 | - '**.yaml' 9 | - '**.md' 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: 1.19 20 | 21 | - name: Test 22 | env: 23 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 24 | run: go test -v ./... 25 | 26 | - name: Build 27 | run: go build -v ./... 28 | 29 | 30 | -------------------------------------------------------------------------------- /apis/azureapi/methods_test.go: -------------------------------------------------------------------------------- 1 | package azureapi 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestListAll(t *testing.T) { 10 | tree, _ := NewMockAzureAPI().GetRepoTree("anubhav06", "testing", "testing", "master", nil) 11 | assert.Equal(t, 12, len(tree.ListAll())) 12 | } 13 | 14 | func TestListAllFiles(t *testing.T) { 15 | tree, _ := NewMockAzureAPI().GetRepoTree("anubhav06", "testing", "testing", "master", nil) 16 | assert.Equal(t, 11, len(tree.ListAllFiles())) 17 | } 18 | 19 | func TestListAllDirs(t *testing.T) { 20 | tree, _ := NewMockAzureAPI().GetRepoTree("anubhav06", "testing", "testing", "master", nil) 21 | assert.Equal(t, 1, len(tree.ListAllDirs())) 22 | } 23 | -------------------------------------------------------------------------------- /bitbucketparser/v1/tree.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import "fmt" 4 | 5 | func (gl *BitBucketURL) ListAllNames() ([]string, error) { 6 | //TODO implement me 7 | return nil, fmt.Errorf("ListAllNames is not supported") 8 | } 9 | 10 | func (gl *BitBucketURL) ListDirsNames() ([]string, error) { 11 | //TODO implement me 12 | return nil, fmt.Errorf("ListDirsNames is not supported") 13 | } 14 | 15 | func (gl *BitBucketURL) ListFilesNames() ([]string, error) { 16 | //TODO implement me 17 | return nil, fmt.Errorf("ListFilesNames is not supported") 18 | } 19 | 20 | func (gl *BitBucketURL) ListFilesNamesWithExtension(extensions []string) ([]string, error) { 21 | //TODO implement me 22 | return nil, fmt.Errorf("ListFilesNamesWithExtension is not supported") 23 | } 24 | -------------------------------------------------------------------------------- /apis/bitbucketapi/datastructures.go: -------------------------------------------------------------------------------- 1 | package bitbucketapi 2 | 3 | import "time" 4 | 5 | type ObjectType string 6 | 7 | type Tree struct{} 8 | 9 | type bitbucketBranchingModel struct { 10 | Development struct { 11 | Name string `json:"name"` 12 | } `json:"development"` 13 | } 14 | 15 | type Headers struct { 16 | Token string 17 | } 18 | 19 | type Commits struct { 20 | Values []Commit `json:"values"` 21 | Pagelen int `json:"pagelen"` 22 | Next string `json:"next"` 23 | } 24 | 25 | type Commit struct { 26 | Type string `json:"type"` 27 | Hash string `json:"hash"` 28 | Date time.Time `json:"date"` 29 | Author struct { 30 | Type string `json:"type"` 31 | Raw string `json:"raw"` 32 | } `json:"author"` 33 | Message string `json:"message"` 34 | } 35 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release-Tag 2 | on: 3 | push: 4 | branches: [ master ] 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | 11 | - name: Set up Go 12 | uses: actions/setup-go@v2 13 | with: 14 | go-version: 1.19 15 | 16 | - name: Build 17 | run: go build -v ./... 18 | 19 | - name: Test 20 | env: 21 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 22 | run: go test -v ./... 23 | 24 | - name: Create a release 25 | id: create_release 26 | uses: actions/create-release@v1 27 | env: 28 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 29 | with: 30 | tag_name: v0.0.${{ github.run_number }} 31 | release_name: Release v0.0.${{ github.run_number }} 32 | draft: false 33 | prerelease: false -------------------------------------------------------------------------------- /apis/gitlabapi/methods.go: -------------------------------------------------------------------------------- 1 | package gitlabapi 2 | 3 | import "fmt" 4 | 5 | // ListAll list all file/dir in repo tree 6 | func (t *Tree) ListAll() []string { 7 | l := []string{} 8 | 9 | for i := range *t { 10 | l = append(l, (*t)[i].Path) 11 | } 12 | return l 13 | } 14 | 15 | // ListAllFiles list all files in repo tree 16 | func (t *Tree) ListAllFiles() []string { 17 | l := []string{} 18 | for i := range *t { 19 | if (*t)[i].Type == ObjectTypeFile { 20 | l = append(l, (*t)[i].Path) 21 | } 22 | } 23 | return l 24 | } 25 | 26 | // ListAllDirs list all directories in repo tree 27 | func (t *Tree) ListAllDirs() []string { 28 | l := []string{} 29 | for i := range *t { 30 | if (*t)[i].Type == ObjectTypeDir { 31 | l = append(l, (*t)[i].Path) 32 | } 33 | } 34 | return l 35 | } 36 | 37 | // ToMap convert headers to map[string]string 38 | func (h *Headers) ToMap() map[string]string { 39 | m := make(map[string]string) 40 | if h.Token != "" { 41 | m["Authorization"] = fmt.Sprintf("Bearer %s", h.Token) 42 | } 43 | return m 44 | } 45 | -------------------------------------------------------------------------------- /apis/githubapi/methods.go: -------------------------------------------------------------------------------- 1 | package githubapi 2 | 3 | import "fmt" 4 | 5 | // ListAll list all file/dir in repo tree 6 | func (t *Tree) ListAll() []string { 7 | l := []string{} 8 | for i := range t.InnerTrees { 9 | l = append(l, t.InnerTrees[i].Path) 10 | } 11 | return l 12 | } 13 | 14 | // ListAllFiles list all files in repo tree 15 | func (t *Tree) ListAllFiles() []string { 16 | l := []string{} 17 | for i := range t.InnerTrees { 18 | if t.InnerTrees[i].Type == ObjectTypeFile { 19 | l = append(l, t.InnerTrees[i].Path) 20 | } 21 | } 22 | return l 23 | } 24 | 25 | // ListAllDirs list all directories in repo tree 26 | func (t *Tree) ListAllDirs() []string { 27 | l := []string{} 28 | for i := range t.InnerTrees { 29 | if t.InnerTrees[i].Type == ObjectTypeDir { 30 | l = append(l, t.InnerTrees[i].Path) 31 | } 32 | } 33 | return l 34 | } 35 | 36 | // ToMap convert headers to map[string]string 37 | func (h *Headers) ToMap() map[string]string { 38 | m := make(map[string]string) 39 | if h.Token != "" { 40 | m["Authorization"] = fmt.Sprintf("token %s", h.Token) 41 | } 42 | return m 43 | } 44 | -------------------------------------------------------------------------------- /githubparser/v1/tree_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/kubescape/go-git-url/apis/githubapi" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func NewGitHubParserWithURLMock(fullURL string) (*GitHubURL, error) { 12 | gh := NewGitHubParserMock() 13 | 14 | if err := gh.Parse(fullURL); err != nil { 15 | return gh, err 16 | } 17 | 18 | return gh, nil 19 | } 20 | 21 | func NewGitHubParserMock() *GitHubURL { 22 | 23 | return &GitHubURL{ 24 | gitHubAPI: githubapi.NewMockGitHubAPI(), 25 | host: githubapi.DEFAULT_HOST, 26 | token: os.Getenv("GITHUB_TOKEN"), 27 | } 28 | } 29 | func TestListAllNames(t *testing.T) { 30 | { 31 | gh, err := NewGitHubParserWithURLMock(urlA) 32 | assert.NoError(t, err) 33 | tree, err := gh.ListAllNames() 34 | assert.NoError(t, err) 35 | assert.Equal(t, 25, len(tree)) 36 | } 37 | } 38 | 39 | func TestGetFileExtension(t *testing.T) { 40 | assert.Equal(t, "yaml", getFileExtension("my/name.yaml")) 41 | assert.Equal(t, "txt", getFileExtension("/my/name.txt")) 42 | assert.Equal(t, "json", getFileExtension("myName.json")) 43 | assert.Equal(t, "", getFileExtension("myName")) 44 | } 45 | -------------------------------------------------------------------------------- /bitbucketparser/v1/commit_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func Test_parseRawUser(t *testing.T) { 10 | tests := []struct { 11 | raw string 12 | wantName string 13 | wantEmail string 14 | }{ 15 | { 16 | raw: "David Wertenteil ", 17 | wantName: "David Wertenteil", 18 | wantEmail: "dwertent@cyberarmor.io", 19 | }, 20 | { 21 | raw: "github-actions[bot] ", 22 | wantName: "github-actions[bot]", 23 | wantEmail: "github-actions[bot]@users.noreply.github.com", 24 | }, 25 | { 26 | raw: "David Wertenteil", 27 | wantName: "David Wertenteil", 28 | wantEmail: "", 29 | }, 30 | { 31 | raw: "", 32 | wantName: "", 33 | wantEmail: "dwertent@cyberarmor.io", 34 | }, 35 | } 36 | for _, tt := range tests { 37 | t.Run(tt.raw, func(t *testing.T) { 38 | got, got1 := parseRawUser(tt.raw) 39 | assert.Equalf(t, tt.wantName, got, "parseRawUser(%v)", tt.raw) 40 | assert.Equalf(t, tt.wantEmail, got1, "parseRawUser(%v)", tt.raw) 41 | }) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /apis/azureapi/methods.go: -------------------------------------------------------------------------------- 1 | package azureapi 2 | 3 | import ( 4 | b64 "encoding/base64" 5 | "fmt" 6 | ) 7 | 8 | // ListAll list all file/dir in repo tree 9 | func (t *Tree) ListAll() []string { 10 | l := []string{} 11 | for i := range t.InnerTree { 12 | l = append(l, t.InnerTree[i].Path) 13 | } 14 | return l 15 | } 16 | 17 | // ListAllFiles list all files in repo tree 18 | func (t *Tree) ListAllFiles() []string { 19 | l := []string{} 20 | for i := range t.InnerTree { 21 | if t.InnerTree[i].GitObjectType == ObjectTypeFile { 22 | l = append(l, t.InnerTree[i].Path) 23 | } 24 | } 25 | return l 26 | } 27 | 28 | // ListAllDirs list all directories in repo tree 29 | func (t *Tree) ListAllDirs() []string { 30 | l := []string{} 31 | for i := range t.InnerTree { 32 | if t.InnerTree[i].GitObjectType == ObjectTypeDir { 33 | l = append(l, t.InnerTree[i].Path) 34 | } 35 | } 36 | return l 37 | } 38 | 39 | // ToMap convert headers to map[string]string 40 | func (h *Headers) ToMap() map[string]string { 41 | m := make(map[string]string) 42 | 43 | if h.Token != "" { 44 | fToken := ":" + h.Token 45 | sEnc := b64.StdEncoding.EncodeToString([]byte(fToken)) 46 | m["Authorization"] = fmt.Sprintf("Basic %s", sEnc) 47 | } 48 | return m 49 | } 50 | -------------------------------------------------------------------------------- /gitlabparser/v1/commit.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/kubescape/go-git-url/apis" 7 | "github.com/kubescape/go-git-url/apis/gitlabapi" 8 | ) 9 | 10 | // ListAll list all directories and files in url tree 11 | func (gl *GitLabURL) GetLatestCommit() (*apis.Commit, error) { 12 | if gl.GetHostName() == "" || gl.GetOwnerName() == "" || gl.GetRepoName() == "" { 13 | return nil, fmt.Errorf("missing host/owner/repo") 14 | } 15 | if gl.GetBranchName() == "" { 16 | if err := gl.SetDefaultBranchName(); err != nil { 17 | return nil, fmt.Errorf("failed to get default branch. reason: %s", err.Error()) 18 | } 19 | } 20 | 21 | c, err := gl.gitLabAPI.GetLatestCommit(gl.GetOwnerName(), gl.GetRepoName(), gl.GetBranchName(), gl.headers()) 22 | if err != nil { 23 | return nil, fmt.Errorf("failed to get latest commit. reason: %s", err.Error()) 24 | } 25 | 26 | return gitLabAPICommitToCommit(c), nil 27 | } 28 | 29 | func gitLabAPICommitToCommit(c *gitlabapi.Commit) *apis.Commit { 30 | latestCommit := &apis.Commit{ 31 | SHA: c.ID, 32 | Author: apis.Committer{ 33 | Name: c.AuthorName, 34 | Email: c.AuthorEmail, 35 | Date: c.AuthoredDate, 36 | }, 37 | Committer: apis.Committer{ 38 | Name: c.CommitterName, 39 | Email: c.CommitterEmail, 40 | Date: c.CommitterDate, 41 | }, 42 | Message: c.Message, 43 | } 44 | 45 | return latestCommit 46 | } 47 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/chainguard-dev/git-urls v1.0.2 h1:pSpT7ifrpc5X55n4aTTm7FFUE+ZQHKiqpiwNkJrVcKQ= 2 | github.com/chainguard-dev/git-urls v1.0.2/go.mod h1:rbGgj10OS7UgZlbzdUQIQpT0k/D4+An04HJY7Ol+Y/o= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= 7 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 8 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 9 | github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= 10 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 11 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 12 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 13 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 14 | k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= 15 | k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= 16 | k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= 17 | -------------------------------------------------------------------------------- /azureparser/v1/commit.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/kubescape/go-git-url/apis" 7 | "github.com/kubescape/go-git-url/apis/azureapi" 8 | ) 9 | 10 | // ListAll list all directories and files in url tree 11 | func (az *AzureURL) GetLatestCommit() (*apis.Commit, error) { 12 | if az.GetHostName() == "" || az.GetOwnerName() == "" || az.GetProjectName() == "" || az.GetRepoName() == "" { 13 | return nil, fmt.Errorf("missing host/owner/project/repo") 14 | } 15 | if az.GetBranchName() == "" { 16 | if err := az.SetDefaultBranchName(); err != nil { 17 | return nil, fmt.Errorf("failed to get default branch. reason: %s", err.Error()) 18 | } 19 | } 20 | 21 | c, err := az.azureAPI.GetLatestCommit(az.GetOwnerName(), az.GetProjectName(), az.GetRepoName(), az.GetBranchName(), az.headers()) 22 | if err != nil { 23 | return nil, fmt.Errorf("failed to get latest commit. reason: %s", err.Error()) 24 | } 25 | 26 | return azureAPICommitToCommit(c), nil 27 | } 28 | 29 | func azureAPICommitToCommit(c *azureapi.Commit) *apis.Commit { 30 | latestCommit := &apis.Commit{ 31 | SHA: c.CommitsValue[0].CommitID, 32 | Author: apis.Committer{ 33 | Name: c.CommitsValue[0].Author.Name, 34 | Email: c.CommitsValue[0].Author.Email, 35 | Date: c.CommitsValue[0].Author.Date, 36 | }, 37 | Committer: apis.Committer{ 38 | Name: c.CommitsValue[0].Committer.Name, 39 | Email: c.CommitsValue[0].Committer.Email, 40 | Date: c.CommitsValue[0].Committer.Date, 41 | }, 42 | Message: c.CommitsValue[0].Comment, 43 | } 44 | 45 | return latestCommit 46 | } 47 | -------------------------------------------------------------------------------- /githubparser/v1/commit.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/kubescape/go-git-url/apis" 7 | "github.com/kubescape/go-git-url/apis/githubapi" 8 | ) 9 | 10 | // ListAll list all directories and files in url tree 11 | func (gh *GitHubURL) GetLatestCommit() (*apis.Commit, error) { 12 | if gh.GetHostName() == "" || gh.GetOwnerName() == "" || gh.GetRepoName() == "" { 13 | return nil, fmt.Errorf("missing host/owner/repo") 14 | } 15 | if gh.GetBranchName() == "" { 16 | if err := gh.SetDefaultBranchName(); err != nil { 17 | return nil, fmt.Errorf("failed to get default branch. reason: %s", err.Error()) 18 | } 19 | } 20 | 21 | c, err := gh.gitHubAPI.GetLatestCommit(gh.GetOwnerName(), gh.GetRepoName(), gh.GetBranchName(), gh.headers()) 22 | if err != nil { 23 | return nil, fmt.Errorf("failed to get latest commit. reason: %s", err.Error()) 24 | } 25 | 26 | return gitHubAPICommitToCommit(c), nil 27 | } 28 | 29 | func gitHubAPICommitToCommit(c *githubapi.Commit) *apis.Commit { 30 | latestCommit := &apis.Commit{ 31 | SHA: c.SHA, 32 | Author: apis.Committer{ 33 | Name: c.Commit.Author.Name, 34 | Email: c.Commit.Author.Email, 35 | Date: c.Commit.Author.Date, 36 | }, 37 | Committer: apis.Committer{ 38 | Name: c.Commit.Committer.Name, 39 | Email: c.Commit.Committer.Email, 40 | Date: c.Commit.Committer.Date, 41 | }, 42 | Message: c.Commit.Message, 43 | } 44 | for i := range c.Files { 45 | latestCommit.Files = append(latestCommit.Files, apis.Files{ 46 | FileSHA: c.Files[i].SHA, 47 | Filename: c.Files[i].Filename, 48 | }) 49 | 50 | } 51 | return latestCommit 52 | } 53 | -------------------------------------------------------------------------------- /apis/gitlabapi/datastructures.go: -------------------------------------------------------------------------------- 1 | package gitlabapi 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type ObjectType string 8 | 9 | const ( 10 | ObjectTypeDir ObjectType = "tree" 11 | ObjectTypeFile ObjectType = "blob" 12 | ) 13 | 14 | type InnerTree struct { 15 | ID string `json:"id"` 16 | Name string `json:"name"` 17 | Type ObjectType `json:"type"` 18 | Path string `json:"path"` 19 | Mode string `json:"mode"` 20 | } 21 | 22 | type Tree []InnerTree 23 | 24 | type gitlabDefaultBranchAPI []struct { 25 | Name string `json:"name"` 26 | Commit Commit `json:"commit"` 27 | Merged bool `json:"merged"` 28 | Protected bool `json:"protected"` 29 | DevelopersCanPush bool `json:"developers_can_push"` 30 | DevelopersCanMerge bool `json:"developers_can_merge"` 31 | CanPush bool `json:"can_push"` 32 | DefaultBranch bool `json:"default"` 33 | WebURL string `json:"web_url"` 34 | } 35 | 36 | type Headers struct { 37 | Token string 38 | } 39 | 40 | // LatestCommit returned structure 41 | type Commit struct { 42 | ID string `json:"id"` 43 | ShortID string `json:"short_id"` 44 | CreatedAt time.Time `json:"created_at"` 45 | Title string `json:"title"` 46 | Message string `json:"message"` 47 | AuthorName string `json:"author_name"` 48 | AuthorEmail string `json:"author_email"` 49 | AuthoredDate time.Time `json:"authored_date"` 50 | CommitterName string `json:"committer_name"` 51 | CommitterEmail string `json:"committer_email"` 52 | CommitterDate time.Time `json:"committed_date"` 53 | WebURL string `json:"web_url"` 54 | ParentIDS []string `json:"parent_ids"` 55 | } -------------------------------------------------------------------------------- /bitbucketparser/v1/commit.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/kubescape/go-git-url/apis" 9 | "github.com/kubescape/go-git-url/apis/bitbucketapi" 10 | ) 11 | 12 | var rawUserRe = regexp.MustCompile("([^<]*)?(<(.+)>)?") 13 | 14 | func (gl *BitBucketURL) GetLatestCommit() (*apis.Commit, error) { 15 | if gl.GetHostName() == "" || gl.GetOwnerName() == "" || gl.GetRepoName() == "" { 16 | return nil, fmt.Errorf("missing host/owner/repo") 17 | } 18 | if gl.GetBranchName() == "" { 19 | if err := gl.SetDefaultBranchName(); err != nil { 20 | return nil, fmt.Errorf("failed to get default branch. reason: %s", err.Error()) 21 | } 22 | } 23 | 24 | c, err := gl.bitBucketAPI.GetLatestCommit(gl.GetOwnerName(), gl.GetRepoName(), gl.GetBranchName(), gl.headers()) 25 | if err != nil { 26 | return nil, fmt.Errorf("failed to get latest commit. reason: %s", err.Error()) 27 | } 28 | 29 | return bitBucketAPICommitToCommit(c), nil 30 | } 31 | 32 | func bitBucketAPICommitToCommit(c *bitbucketapi.Commit) *apis.Commit { 33 | name, email := parseRawUser(c.Author.Raw) 34 | latestCommit := &apis.Commit{ 35 | SHA: c.Hash, 36 | Author: apis.Committer{ 37 | Name: name, 38 | Email: email, 39 | Date: c.Date, 40 | }, 41 | Committer: apis.Committer{ // same as author as API doesn't return the committer 42 | Name: name, 43 | Email: email, 44 | Date: c.Date, 45 | }, 46 | Message: c.Message, 47 | } 48 | 49 | return latestCommit 50 | } 51 | 52 | func parseRawUser(raw string) (string, string) { 53 | match := rawUserRe.FindStringSubmatch(raw) 54 | if match != nil { 55 | return strings.TrimSpace(match[1]), match[3] 56 | } 57 | return raw, "" 58 | } 59 | -------------------------------------------------------------------------------- /interface.go: -------------------------------------------------------------------------------- 1 | package giturl 2 | 3 | import ( 4 | "net/url" 5 | 6 | "github.com/kubescape/go-git-url/apis" 7 | ) 8 | 9 | // IGitURL parse git urls 10 | type IGitURL interface { 11 | SetBranchName(string) 12 | SetOwnerName(string) 13 | SetPath(string) 14 | SetRepoName(string) 15 | 16 | GetHostName() string 17 | GetProvider() string 18 | GetBranchName() string 19 | GetOwnerName() string 20 | GetPath() string 21 | GetRepoName() string 22 | GetHttpCloneURL() string 23 | 24 | // parse url 25 | Parse(fullURL string) error 26 | 27 | // GetURL git url 28 | GetURL() *url.URL 29 | } 30 | 31 | type IGitAPI interface { 32 | IGitURL 33 | 34 | GetToken() string 35 | SetToken(string) 36 | 37 | // set default branch name using the providers git API 38 | SetDefaultBranchName() error 39 | 40 | // ListFilesNamesWithExtension list all files in path with the desired extension. if empty will list all (including directories) 41 | ListFilesNamesWithExtension(extensions []string) ([]string, error) 42 | 43 | // ListAll list all directories and files in url tree 44 | ListAllNames() ([]string, error) 45 | 46 | // ListFilesNames list all files in url tree 47 | ListFilesNames() ([]string, error) 48 | 49 | // ListDirsNames list all directories in url tree 50 | ListDirsNames() ([]string, error) 51 | 52 | // DownloadAllFiles download files from git repo tree 53 | // return map of (url:file, url:error) 54 | DownloadAllFiles() (map[string][]byte, map[string]error) 55 | 56 | // DownloadFilesWithExtension download files from git repo tree based on file extension 57 | // return map of (url:file, url:error) 58 | DownloadFilesWithExtension(extensions []string) (map[string][]byte, map[string]error) 59 | 60 | // GetLatestCommit get latest commit 61 | GetLatestCommit() (*apis.Commit, error) 62 | } 63 | -------------------------------------------------------------------------------- /apis/urlcomposer.go: -------------------------------------------------------------------------------- 1 | package apis 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type UrlComposer interface { 8 | FileUrlByCommit(commit string) string 9 | FileUrlByBranch(branch string) string 10 | FileUrlByTag(tag string) string 11 | } 12 | 13 | type GitHubUrlComposer struct { 14 | remoteUrl, path string 15 | } 16 | 17 | type AzureUrlComposer struct { 18 | remoteUrl, path string 19 | } 20 | 21 | func NewUrlComposer(provider ProviderType, remoteUrl, path string) (UrlComposer, error) { 22 | switch provider { 23 | case ProviderAzure: 24 | return &AzureUrlComposer{remoteUrl: remoteUrl, path: path}, nil 25 | case ProviderGitHub: 26 | return &GitHubUrlComposer{remoteUrl: remoteUrl, path: path}, nil 27 | default: 28 | return nil, fmt.Errorf("unknown provider") 29 | } 30 | } 31 | 32 | func (r *GitHubUrlComposer) fileUrlBase(commitOrBranchOrTag string) string { 33 | return fmt.Sprintf("%s/blob/%s/%s", r.remoteUrl, commitOrBranchOrTag, r.path) 34 | } 35 | 36 | func (r *GitHubUrlComposer) FileUrlByCommit(commit string) string { 37 | return r.fileUrlBase(commit) 38 | } 39 | 40 | func (r *GitHubUrlComposer) FileUrlByBranch(branch string) string { 41 | return r.fileUrlBase(branch) 42 | } 43 | 44 | func (r *GitHubUrlComposer) FileUrlByTag(tag string) string { 45 | return r.fileUrlBase(tag) 46 | } 47 | 48 | func (r *AzureUrlComposer) fileUrlBase(version string) string { 49 | return fmt.Sprintf("%s?path=%s&version=%s", r.remoteUrl, r.path, version) 50 | } 51 | 52 | func (r *AzureUrlComposer) FileUrlByCommit(commit string) string { 53 | return r.fileUrlBase(fmt.Sprintf("GC%s", commit)) 54 | } 55 | 56 | func (r *AzureUrlComposer) FileUrlByBranch(branch string) string { 57 | return r.fileUrlBase(fmt.Sprintf("GB%s", branch)) 58 | } 59 | 60 | func (r *AzureUrlComposer) FileUrlByTag(tag string) string { 61 | return r.fileUrlBase(fmt.Sprintf("GT%s", tag)) 62 | } 63 | -------------------------------------------------------------------------------- /apis/urlcomposer_test.go: -------------------------------------------------------------------------------- 1 | package apis 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | var ( 10 | urlAzure = "https://dev.azure.com/dwertent/ks-testing-public/_git/ks-testing-public" 11 | urlGitHub = "https://github.com/kubescape/go-git-url" 12 | ) 13 | 14 | func TestNewUrlComposer(t *testing.T) { 15 | { 16 | var provider = ProviderType(ProviderAzure.String()) 17 | g, err := NewUrlComposer(provider, urlAzure, "rules-tests/k8s-audit-logs-enabled-native/test-failed/input/apiserverpod.json") 18 | assert.NoError(t, err) 19 | assert.Equal(t, 20 | "https://dev.azure.com/dwertent/ks-testing-public/_git/ks-testing-public?path=rules-tests/k8s-audit-logs-enabled-native/test-failed/input/apiserverpod.json&version=GBmaster", 21 | g.FileUrlByBranch("master")) 22 | assert.Equal(t, 23 | "https://dev.azure.com/dwertent/ks-testing-public/_git/ks-testing-public?path=rules-tests/k8s-audit-logs-enabled-native/test-failed/input/apiserverpod.json&version=GC9300d038ca1be76711aacb2f7ba59b67f4bbddf1", 24 | g.FileUrlByCommit("9300d038ca1be76711aacb2f7ba59b67f4bbddf1")) 25 | assert.Equal(t, 26 | "https://dev.azure.com/dwertent/ks-testing-public/_git/ks-testing-public?path=rules-tests/k8s-audit-logs-enabled-native/test-failed/input/apiserverpod.json&version=GTv1.0.179", 27 | g.FileUrlByTag("v1.0.179")) 28 | } 29 | { 30 | var provider = ProviderType(ProviderGitHub.String()) 31 | g, err := NewUrlComposer(provider, urlGitHub, "files/file0.json") 32 | assert.NoError(t, err) 33 | assert.Equal(t, 34 | "https://github.com/kubescape/go-git-url/blob/master/files/file0.json", 35 | g.FileUrlByBranch("master")) 36 | assert.Equal(t, 37 | "https://github.com/kubescape/go-git-url/blob/c7380d9b941a671253a81c694f644868270a1d81/files/file0.json", 38 | g.FileUrlByCommit("c7380d9b941a671253a81c694f644868270a1d81")) 39 | assert.Equal(t, 40 | "https://github.com/kubescape/go-git-url/blob/v0.0.13/files/file0.json", 41 | g.FileUrlByTag("v0.0.13")) 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /init.go: -------------------------------------------------------------------------------- 1 | package giturl 2 | 3 | import ( 4 | "fmt" 5 | 6 | giturl "github.com/chainguard-dev/git-urls" 7 | 8 | azureparserv1 "github.com/kubescape/go-git-url/azureparser/v1" 9 | bitbucketparserv1 "github.com/kubescape/go-git-url/bitbucketparser/v1" 10 | githubparserv1 "github.com/kubescape/go-git-url/githubparser/v1" 11 | gitlabparserv1 "github.com/kubescape/go-git-url/gitlabparser/v1" 12 | ) 13 | 14 | // NewGitURL get instance of git parser 15 | func NewGitURL(fullURL string) (IGitURL, error) { 16 | hostUrl, err := getHost(fullURL) 17 | if err != nil { 18 | return nil, err 19 | } 20 | 21 | if githubparserv1.IsHostGitHub(hostUrl) { 22 | return githubparserv1.NewGitHubParserWithURL(fullURL) 23 | } 24 | if azureparserv1.IsHostAzure(hostUrl) { 25 | return azureparserv1.NewAzureParserWithURL(fullURL) 26 | } 27 | if bitbucketparserv1.IsHostBitBucket(hostUrl) { 28 | return bitbucketparserv1.NewBitBucketParserWithURL(fullURL) 29 | } 30 | if gitlabparserv1.IsHostGitLab(hostUrl) { 31 | return gitlabparserv1.NewGitLabParserWithURL(hostUrl, fullURL) 32 | } 33 | return nil, fmt.Errorf("repository host '%s' not supported", hostUrl) 34 | } 35 | 36 | // NewGitAPI get instance of git api 37 | func NewGitAPI(fullURL string) (IGitAPI, error) { 38 | hostUrl, err := getHost(fullURL) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | if githubparserv1.IsHostGitHub(hostUrl) { 44 | return githubparserv1.NewGitHubParserWithURL(fullURL) 45 | } 46 | if gitlabparserv1.IsHostGitLab(hostUrl) { 47 | return gitlabparserv1.NewGitLabParserWithURL(hostUrl, fullURL) 48 | } 49 | if azureparserv1.IsHostAzure(hostUrl) { 50 | return azureparserv1.NewAzureParserWithURL(fullURL) 51 | } 52 | if bitbucketparserv1.IsHostBitBucket(hostUrl) { 53 | return bitbucketparserv1.NewBitBucketParserWithURL(fullURL) 54 | } 55 | return nil, fmt.Errorf("repository host '%s' not supported", hostUrl) 56 | } 57 | 58 | func getHost(fullURL string) (string, error) { 59 | parsedURL, err := giturl.Parse(fullURL) 60 | if err != nil { 61 | return "", err 62 | } 63 | 64 | return parsedURL.Host, nil 65 | } 66 | -------------------------------------------------------------------------------- /apis/http.go: -------------------------------------------------------------------------------- 1 | package apis 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "strings" 8 | ) 9 | 10 | // httpGet run http get command and return response as string 11 | func HttpGet(httpClient *http.Client, fullURL string, headers map[string]string) (string, error) { 12 | req, err := http.NewRequest("GET", fullURL, nil) 13 | if err != nil { 14 | return "", err 15 | } 16 | setHeaders(req, headers) 17 | 18 | resp, err := httpClient.Do(req) 19 | if err != nil { 20 | return "", err 21 | } 22 | respStr, err := httpRespToString(resp) 23 | if err != nil { 24 | return "", err 25 | } 26 | return respStr, nil 27 | } 28 | 29 | // HTTPRespToString parses the body as string and checks the HTTP status code, it closes the body reader at the end 30 | func httpRespToString(resp *http.Response) (string, error) { 31 | if resp == nil || resp.Body == nil { 32 | return "", nil 33 | } 34 | strBuilder := strings.Builder{} 35 | defer resp.Body.Close() 36 | if resp.ContentLength > 0 { 37 | strBuilder.Grow(int(resp.ContentLength)) 38 | } 39 | _, err := io.Copy(&strBuilder, resp.Body) 40 | respStr := strBuilder.String() 41 | if err != nil { 42 | respStrNewLen := len(respStr) 43 | if respStrNewLen > 1024 { 44 | respStrNewLen = 1024 45 | } 46 | return "", fmt.Errorf("http-error: '%s', reason: '%s'", resp.Status, respStr[:respStrNewLen]) 47 | // return "", fmt.Errorf("HTTP request failed. URL: '%s', Read-ERROR: '%s', HTTP-CODE: '%s', BODY(top): '%s', HTTP-HEADERS: %v, HTTP-BODY-BUFFER-LENGTH: %v", resp.Request.URL.RequestURI(), err, resp.Status, respStr[:respStrNewLen], resp.Header, bytesNum) 48 | } 49 | if resp.StatusCode < 200 || resp.StatusCode >= 300 { 50 | respStrNewLen := len(respStr) 51 | if respStrNewLen > 1024 { 52 | respStrNewLen = 1024 53 | } 54 | err = fmt.Errorf("http-error: '%s', reason: '%s'", resp.Status, respStr[:respStrNewLen]) 55 | } 56 | 57 | return respStr, err 58 | } 59 | 60 | // set headers for http call 61 | func setHeaders(req *http.Request, headers map[string]string) { 62 | if len(headers) >= 0 { // might be nil 63 | for k, v := range headers { 64 | req.Header.Set(k, v) 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /gitlabparser/v1/download.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | 9 | "github.com/kubescape/go-git-url/apis/gitlabapi" 10 | ) 11 | 12 | // DownloadAllFiles download files from git repo tree 13 | // return map of (url:file, url:error) 14 | func (gl *GitLabURL) DownloadAllFiles() (map[string][]byte, map[string]error) { 15 | 16 | files, err := gl.ListFilesNames() 17 | if err != nil { 18 | return nil, map[string]error{"": err} // TODO - update error 19 | } 20 | return downloadFiles(gl.pathsToURLs(files)) 21 | } 22 | 23 | // DownloadFilesWithExtension download files from git repo tree based on file extension 24 | // return map of (url:file, url:error) 25 | func (gl *GitLabURL) DownloadFilesWithExtension(filesExtensions []string) (map[string][]byte, map[string]error) { 26 | files, err := gl.ListFilesNamesWithExtension(filesExtensions) 27 | if err != nil { 28 | return nil, map[string]error{"": err} // TODO - update error 29 | } 30 | 31 | return downloadFiles(gl.pathsToURLs(files)) 32 | } 33 | 34 | // DownloadFiles download files from git repo. Call ListAllNames/ListFilesNamesWithExtention before calling this function 35 | // return map of (url:file, url:error) 36 | func downloadFiles(urls []string) (map[string][]byte, map[string]error) { 37 | var errs map[string]error 38 | files := map[string][]byte{} 39 | for i := range urls { 40 | 41 | file, err := downloadFile(urls[i]) 42 | if err != nil { 43 | if errs == nil { 44 | errs = map[string]error{} 45 | } 46 | errs[urls[i]] = err 47 | continue 48 | } 49 | files[urls[i]] = file 50 | } 51 | return files, errs 52 | } 53 | 54 | // download single file 55 | func downloadFile(url string) ([]byte, error) { 56 | resp, err := http.Get(url) 57 | if err != nil { 58 | return nil, err 59 | } 60 | defer resp.Body.Close() 61 | if resp.StatusCode < 200 || 301 < resp.StatusCode { 62 | return nil, fmt.Errorf("failed to download file, url: '%s', status code: %s", url, resp.Status) 63 | } 64 | return streamToByte(resp.Body) 65 | } 66 | 67 | func streamToByte(stream io.Reader) ([]byte, error) { 68 | buf := new(bytes.Buffer) 69 | if _, err := buf.ReadFrom(stream); err != nil { 70 | return nil, err 71 | } 72 | return buf.Bytes(), nil 73 | } 74 | 75 | func (gl *GitLabURL) pathsToURLs(files []string) []string { 76 | var rawURLs []string 77 | for i := range files { 78 | rawURLs = append(rawURLs, gitlabapi.APIRaw(gl.host, gl.GetOwnerName(), gl.GetRepoName(), files[i])) 79 | } 80 | return rawURLs 81 | } 82 | -------------------------------------------------------------------------------- /githubparser/v1/download.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | 9 | "github.com/kubescape/go-git-url/apis/githubapi" 10 | ) 11 | 12 | // DownloadAllFiles download files from git repo tree 13 | // return map of (url:file, url:error) 14 | func (gh *GitHubURL) DownloadAllFiles() (map[string][]byte, map[string]error) { 15 | files, err := gh.ListFilesNames() 16 | if err != nil { 17 | return nil, map[string]error{"": err} // TODO - update error 18 | } 19 | return downloadFiles(gh.pathsToURLs(files)) 20 | } 21 | 22 | // DownloadFilesWithExtension download files from git repo tree based on file extension 23 | // return map of (url:file, url:error) 24 | func (gh *GitHubURL) DownloadFilesWithExtension(filesExtensions []string) (map[string][]byte, map[string]error) { 25 | files, err := gh.ListFilesNamesWithExtension(filesExtensions) 26 | if err != nil { 27 | return nil, map[string]error{"": err} // TODO - update error 28 | } 29 | 30 | return downloadFiles(gh.pathsToURLs(files)) 31 | } 32 | 33 | // DownloadFiles download files from git repo. Call ListAllNames/ListFilesNamesWithExtention before calling this function 34 | // return map of (url:file, url:error) 35 | func downloadFiles(urls []string) (map[string][]byte, map[string]error) { 36 | var errs map[string]error 37 | files := map[string][]byte{} 38 | for i := range urls { 39 | 40 | file, err := downloadFile(urls[i]) 41 | if err != nil { 42 | if errs == nil { 43 | errs = map[string]error{} 44 | } 45 | errs[urls[i]] = err 46 | continue 47 | } 48 | files[urls[i]] = file 49 | } 50 | return files, errs 51 | } 52 | 53 | // download single file 54 | func downloadFile(url string) ([]byte, error) { 55 | resp, err := http.Get(url) 56 | if err != nil { 57 | return nil, err 58 | } 59 | defer resp.Body.Close() 60 | if resp.StatusCode < 200 || 301 < resp.StatusCode { 61 | return nil, fmt.Errorf("failed to download file, url: '%s', status code: %s", url, resp.Status) 62 | } 63 | return streamToByte(resp.Body) 64 | } 65 | 66 | func streamToByte(stream io.Reader) ([]byte, error) { 67 | buf := new(bytes.Buffer) 68 | if _, err := buf.ReadFrom(stream); err != nil { 69 | return nil, err 70 | } 71 | return buf.Bytes(), nil 72 | } 73 | 74 | func (gh *GitHubURL) pathsToURLs(files []string) []string { 75 | var rawURLs []string 76 | for i := range files { 77 | rawURLs = append(rawURLs, githubapi.APIRaw(gh.GetOwnerName(), gh.GetRepoName(), gh.GetBranchName(), files[i])) 78 | } 79 | return rawURLs 80 | } 81 | -------------------------------------------------------------------------------- /azureparser/v1/download.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | 9 | "github.com/kubescape/go-git-url/apis/azureapi" 10 | ) 11 | 12 | // DownloadAllFiles download files from git repo tree 13 | // return map of (url:file, url:error) 14 | func (az *AzureURL) DownloadAllFiles() (map[string][]byte, map[string]error) { 15 | 16 | files, err := az.ListFilesNames() 17 | if err != nil { 18 | return nil, map[string]error{"": err} // TODO - update error 19 | } 20 | return downloadFiles(az.pathsToURLs(files)) 21 | } 22 | 23 | // DownloadFilesWithExtension download files from git repo tree based on file extension 24 | // return map of (url:file, url:error) 25 | func (az *AzureURL) DownloadFilesWithExtension(filesExtensions []string) (map[string][]byte, map[string]error) { 26 | files, err := az.ListFilesNamesWithExtension(filesExtensions) 27 | if err != nil { 28 | return nil, map[string]error{"": err} // TODO - update error 29 | } 30 | 31 | return downloadFiles(az.pathsToURLs(files)) 32 | } 33 | 34 | // DownloadFiles download files from git repo. Call ListAllNames/ListFilesNamesWithExtention before calling this function 35 | // return map of (url:file, url:error) 36 | func downloadFiles(urls []string) (map[string][]byte, map[string]error) { 37 | var errs map[string]error 38 | files := map[string][]byte{} 39 | for i := range urls { 40 | 41 | file, err := downloadFile(urls[i]) 42 | if err != nil { 43 | if errs == nil { 44 | errs = map[string]error{} 45 | } 46 | errs[urls[i]] = err 47 | continue 48 | } 49 | files[urls[i]] = file 50 | } 51 | return files, errs 52 | } 53 | 54 | // download single file 55 | func downloadFile(url string) ([]byte, error) { 56 | resp, err := http.Get(url) 57 | if err != nil { 58 | return nil, err 59 | } 60 | defer resp.Body.Close() 61 | if resp.StatusCode < 200 || 301 < resp.StatusCode { 62 | return nil, fmt.Errorf("failed to download file, url: '%s', status code: %s", url, resp.Status) 63 | } 64 | return streamToByte(resp.Body) 65 | } 66 | 67 | func streamToByte(stream io.Reader) ([]byte, error) { 68 | buf := new(bytes.Buffer) 69 | if _, err := buf.ReadFrom(stream); err != nil { 70 | return nil, err 71 | } 72 | return buf.Bytes(), nil 73 | } 74 | 75 | func (az *AzureURL) pathsToURLs(files []string) []string { 76 | var rawURLs []string 77 | for i := range files { 78 | rawURLs = append(rawURLs, azureapi.APIRaw(az.GetOwnerName(), az.GetProjectName(), az.GetRepoName(), az.GetBranchName(), files[i])) 79 | } 80 | return rawURLs 81 | } 82 | -------------------------------------------------------------------------------- /apis/azureapi/datastructures.go: -------------------------------------------------------------------------------- 1 | package azureapi 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type ObjectType string 8 | 9 | const ( 10 | ObjectTypeDir ObjectType = "tree" 11 | ObjectTypeFile ObjectType = "blob" 12 | ) 13 | 14 | type InnerTree struct { 15 | ObjectID string `json:"objectId"` 16 | GitObjectType ObjectType `json:"gitObjectType"` 17 | CommitID string `json:"commitId"` 18 | Path string `json:"path"` 19 | IsFolder bool `json:"isFolder"` 20 | Url string `json:"url"` 21 | } 22 | 23 | type Tree struct { 24 | Count int `json:"count"` 25 | InnerTree []InnerTree `json:"value"` 26 | } 27 | 28 | type CommitsMetadata struct { 29 | TreeID string `json:"treeId"` 30 | CommitID string `json:"commitId"` 31 | Author struct { 32 | Name string `json:"name"` 33 | Email string `json:"email"` 34 | Date time.Time `json:"date"` 35 | } `json:"author"` 36 | Committer struct { 37 | Name string `json:"name"` 38 | Email string `json:"email"` 39 | Date time.Time `json:"date"` 40 | } `json:"committer"` 41 | Comment string `json:"comment"` 42 | Parents []string `json:"parents"` 43 | URL string `json:"url"` 44 | } 45 | 46 | type BranchValue struct { 47 | Commit CommitsMetadata `json:"commit"` 48 | Name string `json:"name"` 49 | AheadCount int `json:"aheadCount"` 50 | BehindCount int `json:"behindCount"` 51 | IsBaseVersion bool `json:"isBaseVersion"` 52 | } 53 | 54 | type azureDefaultBranchAPI struct { 55 | Count int `json:"count"` 56 | BranchValue []BranchValue `json:"value"` 57 | } 58 | 59 | type Headers struct { 60 | Token string 61 | } 62 | 63 | type CommitsValue struct { 64 | CommitID string `json:"commitId"` 65 | Author struct { 66 | Name string `json:"name"` 67 | Email string `json:"email"` 68 | Date time.Time `json:"date"` 69 | } `json:"author"` 70 | Committer struct { 71 | Name string `json:"name"` 72 | Email string `json:"email"` 73 | Date time.Time `json:"date"` 74 | } `json:"committer"` 75 | Comment string `json:"comment"` 76 | ChangeCounts struct { 77 | Add int `json:"add"` 78 | Edit int `json:"edit"` 79 | Delete int `json:"delete"` 80 | } `json:"changeCounts"` 81 | Changes []struct { 82 | SourceServerItem string `json:"sourceServerItem"` 83 | ChangeType string `json:"changeType"` 84 | } `json:"changes"` 85 | URL string `json:"url"` 86 | RemoteURL string `json:"remoteUrl"` 87 | } 88 | 89 | // LatestCommit returned structure 90 | type Commit struct { 91 | Count int `json:"count"` 92 | CommitsValue []CommitsValue `json:"value"` 93 | } 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GIT Parser 2 | 3 | The `git-parser` is a package meant for parsing git urls 4 | 5 | This package also enables listing all files based on there extension 6 | 7 | ## Parser 8 | 9 | ### Supported parsers 10 | 11 | * GitHub 12 | * GitLab 13 | * Azure 14 | 15 | ### Parse a git URL 16 | 17 | ```go 18 | package main 19 | 20 | import ( 21 | "fmt" 22 | 23 | giturl "github.com/kubescape/go-git-url" 24 | ) 25 | 26 | func main() { 27 | 28 | fullURl := "https://github.com/kubescape/go-git-url" 29 | gitURL, err := giturl.NewGitURL(fullURl) // initialize and parse the URL 30 | if err != nil { 31 | // do something 32 | } 33 | 34 | fmt.Printf(gitURL.GetHostName()) // github.com 35 | fmt.Printf(gitURL.GetOwnerName()) // kubescape 36 | fmt.Printf(gitURL.GetRepoName()) // go-git-url 37 | } 38 | ``` 39 | 40 | ## Git API support 41 | 42 | ### Supported APIs 43 | 44 | * GitHub 45 | > It is recommended to use a [GitHub token](https://docs.github.com/en/enterprise-server@3.4/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token). Set the GitHub token in the `GITHUB_TOKEN` env 46 | 47 | * GitLab 48 | > It is recommended to use a [GitLab token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token). Set the GitLab token in the `GITLAB_TOKEN` env 49 | 50 | * Azure 51 | > It is recommended to use a [Azure token](https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=Windows#create-a-pat). Set the Azure token in the `AZURE_TOKEN` env 52 | 53 | ### List files and directories 54 | ```go 55 | 56 | // List all files and directories names 57 | all, err := gitURL.ListAllNames() 58 | 59 | // List all files names 60 | files, err := gitURL.ListFilesNames() 61 | 62 | // List all directories names 63 | dirs, err := gitURL.ListDirsNames() 64 | 65 | // List files names with the listed extensions 66 | extensions := []string{"yaml", "json"} 67 | files, err := gitURL.ListFilesNamesWithExtension(extensions) 68 | 69 | ``` 70 | 71 | Different URL support -> 72 | ```go 73 | 74 | basicURL, err := giturl.NewGitURL("https://github.com/kubescape/go-git-url") 75 | 76 | nestedURL, err := giturl.NewGitURL("https://github.com/kubescape/go-git-url/tree/master/files") 77 | 78 | fileApiURL, err := giturl.NewGitURL("https://github.com/kubescape/go-git-url/blob/master/files/file0.json") 79 | 80 | fileRawURL, err := giturl.NewGitURL("https://raw.githubusercontent.com/kubescape/go-git-url/master/files/file0.json") 81 | 82 | ``` 83 | ### Download files 84 | ```go 85 | 86 | // Download all files 87 | all, err := gitURL.DownloadAllFiles() 88 | 89 | // Download all files with the listed extensions 90 | extensions := []string{"yaml", "json"} 91 | files, err := gitURL.DownloadFilesWithExtension(extensions) 92 | 93 | ``` 94 | -------------------------------------------------------------------------------- /gitlabparser/v1/tree.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "strings" 7 | 8 | "github.com/kubescape/go-git-url/apis/gitlabapi" 9 | "k8s.io/utils/strings/slices" 10 | ) 11 | 12 | // ListAll list all directories and files in url tree 13 | func (gl *GitLabURL) GetTree() (*gitlabapi.Tree, error) { 14 | 15 | if gl.isFile { 16 | return &gitlabapi.Tree{ 17 | gitlabapi.InnerTree{ 18 | Path: gl.GetPath(), 19 | Type: gitlabapi.ObjectTypeFile, 20 | }, 21 | }, nil 22 | } 23 | 24 | if gl.GetHostName() == "" || gl.GetOwnerName() == "" || gl.GetRepoName() == "" { 25 | return nil, fmt.Errorf("missing host/owner/repo") 26 | } 27 | 28 | if gl.GetBranchName() == "" { 29 | if err := gl.SetDefaultBranchName(); err != nil { 30 | return nil, fmt.Errorf("failed to get default branch. reason: %s", err.Error()) 31 | } 32 | } 33 | 34 | repoTree, err := gl.gitLabAPI.GetRepoTree(gl.GetOwnerName(), gl.GetRepoName(), gl.GetBranchName(), gl.headers()) 35 | if err != nil { 36 | return repoTree, fmt.Errorf("failed to get repo tree. reason: %s", err.Error()) 37 | } 38 | 39 | return repoTree, nil 40 | } 41 | 42 | // ListAllNames list all directories and files in url tree 43 | func (gl *GitLabURL) ListAllNames() ([]string, error) { 44 | repoTree, err := gl.GetTree() 45 | if err != nil { 46 | return []string{}, fmt.Errorf("failed to get repo tree. reason: %s", err.Error()) 47 | } 48 | return repoTree.ListAll(), nil 49 | } 50 | 51 | // ListDirsNames list all directories in url tree 52 | func (gl *GitLabURL) ListDirsNames() ([]string, error) { 53 | repoTree, err := gl.GetTree() 54 | if err != nil { 55 | return []string{}, fmt.Errorf("failed to get repo tree. reason: %s", err.Error()) 56 | } 57 | 58 | return repoTree.ListAllDirs(), nil 59 | } 60 | 61 | 62 | // ListAll list all files in url tree 63 | func (gl *GitLabURL) ListFilesNames() ([]string, error) { 64 | repoTree, err := gl.GetTree() 65 | if err != nil { 66 | return []string{}, fmt.Errorf("failed to get repo tree. reason: %s", err.Error()) 67 | } 68 | 69 | return repoTree.ListAllFiles(), nil 70 | } 71 | 72 | // ListFilesNamesWithExtension list all files in path with the desired extension 73 | func (gl *GitLabURL) ListFilesNamesWithExtension(filesExtensions []string) ([]string, error) { 74 | 75 | fileNames, err := gl.ListFilesNames() 76 | if err != nil { 77 | return []string{}, err 78 | } 79 | if len(fileNames) == 0 { 80 | return fileNames, nil 81 | } 82 | 83 | var files []string 84 | for _, path := range fileNames { 85 | if gl.GetPath() != "" && !strings.HasPrefix(path, gl.GetPath()) { 86 | continue 87 | } 88 | if slices.Contains(filesExtensions, getFileExtension(path)) { 89 | files = append(files, path) 90 | } 91 | 92 | } 93 | return files, nil 94 | } 95 | 96 | func getFileExtension(path string) string { 97 | return strings.TrimPrefix(filepath.Ext(path), ".") 98 | } 99 | -------------------------------------------------------------------------------- /apis/bitbucketapi/apis.go: -------------------------------------------------------------------------------- 1 | package bitbucketapi 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | 9 | "github.com/kubescape/go-git-url/apis" 10 | ) 11 | 12 | const ( 13 | DEFAULT_HOST string = "bitbucket.org" 14 | ) 15 | 16 | type IBitBucketAPI interface { 17 | GetRepoTree(owner, repo, branch string, headers *Headers) (*Tree, error) 18 | GetDefaultBranchName(owner, repo string, headers *Headers) (string, error) 19 | GetLatestCommit(owner, repo, branch string, headers *Headers) (*Commit, error) 20 | } 21 | 22 | type BitBucketAPI struct { 23 | httpClient *http.Client 24 | } 25 | 26 | func NewBitBucketAPI() *BitBucketAPI { return &BitBucketAPI{httpClient: &http.Client{}} } 27 | 28 | func (gl *BitBucketAPI) GetRepoTree(owner, repo, branch string, headers *Headers) (*Tree, error) { 29 | //TODO implement me 30 | return nil, fmt.Errorf("GetRepoTree is not supported") 31 | } 32 | 33 | func (gl *BitBucketAPI) GetDefaultBranchName(owner, repo string, headers *Headers) (string, error) { 34 | body, err := apis.HttpGet(gl.httpClient, APIBranchingModel(owner, repo), headers.ToMap()) 35 | if err != nil { 36 | return "", err 37 | } 38 | 39 | var data bitbucketBranchingModel 40 | if err = json.Unmarshal([]byte(body), &data); err != nil { 41 | return "", err 42 | } 43 | 44 | return data.Development.Name, nil 45 | } 46 | 47 | func (gl *BitBucketAPI) GetLatestCommit(owner, repo, branch string, headers *Headers) (*Commit, error) { 48 | body, err := apis.HttpGet(gl.httpClient, APILastCommitsOfBranch(owner, repo, branch), headers.ToMap()) 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | var data Commits 54 | err = json.Unmarshal([]byte(body), &data) 55 | if err != nil { 56 | return nil, err 57 | } 58 | return &data.Values[0], nil 59 | } 60 | 61 | // APIBranchingModel 62 | // API Ref: https://developer.atlassian.com/cloud/bitbucket/rest/api-group-branching-model/#api-group-branching-model 63 | // Example: https://bitbucket.org/!api/2.0/repositories/matthyx/ks-testing-public/branching-model 64 | func APIBranchingModel(owner, repo string) string { 65 | p, _ := url.JoinPath("!api/2.0/repositories", owner, repo, "branching-model") 66 | u := url.URL{ 67 | Scheme: "https", 68 | Host: DEFAULT_HOST, 69 | Path: p, 70 | } 71 | return u.String() 72 | } 73 | 74 | // APILastCommitsOfBranch 75 | // API Ref: https://developer.atlassian.com/cloud/bitbucket/rest/api-group-commits/#api-repositories-workspace-repo-slug-commits-get 76 | // Example: https://bitbucket.org/!api/2.0/repositories/matthyx/ks-testing-public/commits/?include=master 77 | func APILastCommitsOfBranch(owner, repo, branch string) string { 78 | p, _ := url.JoinPath("!api/2.0/repositories", owner, repo, "commits") 79 | u := url.URL{ 80 | Scheme: "https", 81 | Host: DEFAULT_HOST, 82 | Path: p, 83 | RawQuery: fmt.Sprintf("include=%s", branch), 84 | } 85 | return u.String() 86 | } 87 | -------------------------------------------------------------------------------- /githubparser/v1/tree.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "strings" 7 | 8 | "github.com/kubescape/go-git-url/apis/githubapi" 9 | "k8s.io/utils/strings/slices" 10 | ) 11 | 12 | // ListAll list all directories and files in url tree 13 | func (gh *GitHubURL) GetTree() (*githubapi.Tree, error) { 14 | if gh.isFile { 15 | return &githubapi.Tree{ 16 | InnerTrees: []githubapi.InnerTree{ 17 | { 18 | Path: gh.GetPath(), 19 | Type: githubapi.ObjectTypeFile, 20 | }, 21 | }, 22 | }, nil 23 | } 24 | if gh.GetHostName() == "" || gh.GetOwnerName() == "" || gh.GetRepoName() == "" { 25 | return nil, fmt.Errorf("missing host/owner/repo") 26 | } 27 | if gh.GetBranchName() == "" { 28 | if err := gh.SetDefaultBranchName(); err != nil { 29 | return nil, fmt.Errorf("failed to get default branch. reason: %s", err.Error()) 30 | } 31 | } 32 | 33 | repoTree, err := gh.gitHubAPI.GetRepoTree(gh.GetOwnerName(), gh.GetRepoName(), gh.GetBranchName(), gh.headers()) 34 | if err != nil { 35 | return repoTree, fmt.Errorf("failed to get repo tree. reason: %s", err.Error()) 36 | } 37 | 38 | return repoTree, nil 39 | } 40 | 41 | // ListAllNames list all directories and files in url tree 42 | func (gh *GitHubURL) ListAllNames() ([]string, error) { 43 | repoTree, err := gh.GetTree() 44 | if err != nil { 45 | return []string{}, fmt.Errorf("failed to get repo tree. reason: %s", err.Error()) 46 | } 47 | return repoTree.ListAll(), nil 48 | } 49 | 50 | // ListDirsNames list all directories in url tree 51 | func (gh *GitHubURL) ListDirsNames() ([]string, error) { 52 | repoTree, err := gh.GetTree() 53 | if err != nil { 54 | return []string{}, fmt.Errorf("failed to get repo tree. reason: %s", err.Error()) 55 | } 56 | 57 | return repoTree.ListAllDirs(), nil 58 | } 59 | 60 | // ListAll list all files in url tree 61 | func (gh *GitHubURL) ListFilesNames() ([]string, error) { 62 | repoTree, err := gh.GetTree() 63 | if err != nil { 64 | return []string{}, fmt.Errorf("failed to get repo tree. reason: %s", err.Error()) 65 | } 66 | 67 | return repoTree.ListAllFiles(), nil 68 | } 69 | 70 | // ListFilesNamesWithExtension list all files in path with the desired extension 71 | func (gh *GitHubURL) ListFilesNamesWithExtension(filesExtensions []string) ([]string, error) { 72 | 73 | fileNames, err := gh.ListFilesNames() 74 | if err != nil { 75 | return []string{}, err 76 | } 77 | if len(fileNames) == 0 { 78 | return fileNames, nil 79 | } 80 | 81 | var files []string 82 | for _, path := range fileNames { 83 | if gh.GetPath() != "" && !strings.HasPrefix(path, gh.GetPath()) { 84 | continue 85 | } 86 | if slices.Contains(filesExtensions, getFileExtension(path)) { 87 | files = append(files, path) 88 | } 89 | 90 | } 91 | return files, nil 92 | } 93 | 94 | func getFileExtension(path string) string { 95 | return strings.TrimPrefix(filepath.Ext(path), ".") 96 | } 97 | -------------------------------------------------------------------------------- /azureparser/v1/tree.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "strings" 7 | 8 | "github.com/kubescape/go-git-url/apis/azureapi" 9 | "k8s.io/utils/strings/slices" 10 | ) 11 | 12 | // ListAll list all directories and files in url tree 13 | func (az *AzureURL) GetTree() (*azureapi.Tree, error) { 14 | 15 | if az.isFile { 16 | return &azureapi.Tree{ 17 | InnerTree: []azureapi.InnerTree{ 18 | { 19 | Path: az.GetPath(), 20 | GitObjectType: azureapi.ObjectTypeFile, 21 | }, 22 | }, 23 | }, nil 24 | } 25 | 26 | if az.GetHostName() == "" || az.GetOwnerName() == "" || az.GetProjectName() == "" || az.GetRepoName() == "" { 27 | return nil, fmt.Errorf("missing host/owner/project/repo") 28 | } 29 | 30 | if az.GetBranchName() == "" { 31 | if err := az.SetDefaultBranchName(); err != nil { 32 | return nil, fmt.Errorf("failed to get default branch. reason: %s", err.Error()) 33 | } 34 | } 35 | 36 | repoTree, err := az.azureAPI.GetRepoTree(az.GetOwnerName(), az.GetProjectName(), az.GetRepoName(), az.GetBranchName(), az.headers()) 37 | if err != nil { 38 | return repoTree, fmt.Errorf("failed to get repo tree. reason: %s", err.Error()) 39 | } 40 | 41 | return repoTree, nil 42 | } 43 | 44 | // ListAllNames list all directories and files in url tree 45 | func (az *AzureURL) ListAllNames() ([]string, error) { 46 | 47 | repoTree, err := az.GetTree() 48 | 49 | if err != nil { 50 | return []string{}, fmt.Errorf("failed to get repo tree. reason: %s", err.Error()) 51 | } 52 | 53 | return repoTree.ListAll(), nil 54 | } 55 | 56 | // ListDirsNames list all directories in url tree 57 | func (az *AzureURL) ListDirsNames() ([]string, error) { 58 | repoTree, err := az.GetTree() 59 | if err != nil { 60 | return []string{}, fmt.Errorf("failed to get repo tree. reason: %s", err.Error()) 61 | } 62 | 63 | return repoTree.ListAllDirs(), nil 64 | } 65 | 66 | // ListAll list all files in url tree 67 | func (az *AzureURL) ListFilesNames() ([]string, error) { 68 | repoTree, err := az.GetTree() 69 | if err != nil { 70 | return []string{}, fmt.Errorf("failed to get repo tree. reason: %s", err.Error()) 71 | } 72 | 73 | return repoTree.ListAllFiles(), nil 74 | } 75 | 76 | // ListFilesNamesWithExtension list all files in path with the desired extension 77 | func (az *AzureURL) ListFilesNamesWithExtension(filesExtensions []string) ([]string, error) { 78 | 79 | fileNames, err := az.ListFilesNames() 80 | if err != nil { 81 | return []string{}, err 82 | } 83 | if len(fileNames) == 0 { 84 | return fileNames, nil 85 | } 86 | 87 | var files []string 88 | for _, path := range fileNames { 89 | if az.GetPath() != "" && !strings.HasPrefix(path, az.GetPath()) { 90 | continue 91 | } 92 | if slices.Contains(filesExtensions, getFileExtension(path)) { 93 | files = append(files, path) 94 | } 95 | 96 | } 97 | return files, nil 98 | } 99 | 100 | func getFileExtension(path string) string { 101 | return strings.TrimPrefix(filepath.Ext(path), ".") 102 | } 103 | -------------------------------------------------------------------------------- /githubparser/v1/parser.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "os" 7 | "strings" 8 | 9 | giturl "github.com/chainguard-dev/git-urls" 10 | "github.com/kubescape/go-git-url/apis" 11 | "github.com/kubescape/go-git-url/apis/githubapi" 12 | ) 13 | 14 | // NewGitHubParser empty instance of a github parser 15 | func NewGitHubParser() *GitHubURL { 16 | 17 | return &GitHubURL{ 18 | gitHubAPI: githubapi.NewGitHubAPI(), 19 | host: githubapi.DEFAULT_HOST, 20 | token: os.Getenv("GITHUB_TOKEN"), 21 | } 22 | } 23 | 24 | // NewGitHubParserWithURL parsed instance of a github parser 25 | func NewGitHubParserWithURL(fullURL string) (*GitHubURL, error) { 26 | gh := NewGitHubParser() 27 | 28 | if err := gh.Parse(fullURL); err != nil { 29 | return gh, err 30 | } 31 | 32 | return gh, nil 33 | } 34 | 35 | func (gh *GitHubURL) GetURL() *url.URL { 36 | return &url.URL{ 37 | Scheme: "https", 38 | Host: gh.GetHostName(), 39 | Path: fmt.Sprintf("%s/%s", gh.GetOwnerName(), gh.GetRepoName()), 40 | } 41 | } 42 | func IsHostGitHub(host string) bool { 43 | return host == githubapi.DEFAULT_HOST || host == githubapi.RAW_HOST || host == githubapi.SUBDOMAIN_HOST 44 | } 45 | 46 | func (gh *GitHubURL) GetProvider() string { return apis.ProviderGitHub.String() } 47 | func (gh *GitHubURL) GetHostName() string { return gh.host } 48 | func (gh *GitHubURL) GetBranchName() string { return gh.branch } 49 | func (gh *GitHubURL) GetOwnerName() string { return gh.owner } 50 | func (gh *GitHubURL) GetRepoName() string { return gh.repo } 51 | func (gh *GitHubURL) GetPath() string { return gh.path } 52 | func (gh *GitHubURL) GetToken() string { return gh.token } 53 | func (gh *GitHubURL) GetHttpCloneURL() string { 54 | return fmt.Sprintf("https://github.com/%s/%s.git", gh.GetOwnerName(), gh.GetRepoName()) 55 | } 56 | 57 | func (gh *GitHubURL) SetOwnerName(o string) { gh.owner = o } 58 | func (gh *GitHubURL) SetRepoName(r string) { gh.repo = r } 59 | func (gh *GitHubURL) SetBranchName(branch string) { gh.branch = branch } 60 | func (gh *GitHubURL) SetPath(p string) { gh.path = p } 61 | func (gh *GitHubURL) SetToken(token string) { gh.token = token } 62 | 63 | // Parse URL 64 | func (gh *GitHubURL) Parse(fullURL string) error { 65 | parsedURL, err := giturl.Parse(fullURL) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | if parsedURL.Host == githubapi.RAW_HOST { 71 | gh.isFile = true 72 | } 73 | 74 | index := 0 75 | 76 | splittedRepo := strings.FieldsFunc(parsedURL.Path, func(c rune) bool { return c == '/' }) // trim empty fields from returned slice 77 | if len(splittedRepo) < 2 { 78 | return fmt.Errorf("expecting / in url path, received: '%s'", parsedURL.Path) 79 | } 80 | gh.owner = splittedRepo[index] 81 | index += 1 82 | gh.repo = strings.TrimSuffix(splittedRepo[index], ".git") 83 | index += 1 84 | 85 | // root of repo 86 | if len(splittedRepo) < index+1 { 87 | return nil 88 | } 89 | 90 | // is file or dir 91 | switch splittedRepo[index] { 92 | case "blob": 93 | gh.isFile = true 94 | index += 1 95 | case "tree": 96 | gh.isFile = false 97 | index += 1 98 | } 99 | 100 | if len(splittedRepo) < index+1 { 101 | return nil 102 | } 103 | 104 | gh.branch = splittedRepo[index] 105 | index += 1 106 | 107 | if len(splittedRepo) < index+1 { 108 | return nil 109 | } 110 | gh.path = strings.Join(splittedRepo[index:], "/") 111 | 112 | return nil 113 | } 114 | 115 | // Set the default brach of the repo 116 | func (gh *GitHubURL) SetDefaultBranchName() error { 117 | branch, err := gh.gitHubAPI.GetDefaultBranchName(gh.GetOwnerName(), gh.GetRepoName(), gh.headers()) 118 | if err != nil { 119 | return err 120 | } 121 | gh.branch = branch 122 | return nil 123 | } 124 | 125 | func (gh *GitHubURL) headers() *githubapi.Headers { 126 | return &githubapi.Headers{Token: gh.GetToken()} 127 | } 128 | -------------------------------------------------------------------------------- /gitlabparser/v1/parser.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "os" 7 | "strings" 8 | 9 | giturl "github.com/chainguard-dev/git-urls" 10 | "github.com/kubescape/go-git-url/apis" 11 | "github.com/kubescape/go-git-url/apis/gitlabapi" 12 | ) 13 | 14 | // NewGitHubParserWithURL parsed instance of a github parser 15 | func NewGitLabParserWithURL(host, fullURL string) (*GitLabURL, error) { 16 | gl := &GitLabURL{ 17 | gitLabAPI: gitlabapi.NewGitLabAPI(host), 18 | token: os.Getenv("GITLAB_TOKEN"), 19 | } 20 | 21 | if err := gl.Parse(fullURL); err != nil { 22 | return nil, err 23 | } 24 | 25 | return gl, nil 26 | } 27 | 28 | func (gl *GitLabURL) GetURL() *url.URL { 29 | return &url.URL{ 30 | Scheme: "https", 31 | Host: gl.GetHostName(), 32 | Path: fmt.Sprintf("%s/%s", gl.GetOwnerName(), gl.GetRepoName()), 33 | } 34 | } 35 | 36 | func IsHostGitLab(host string) bool { 37 | return strings.Contains(host, "gitlab") || strings.HasPrefix(host, "git.") 38 | } 39 | 40 | func (gl *GitLabURL) GetProvider() string { return apis.ProviderGitLab.String() } 41 | func (gl *GitLabURL) GetHostName() string { return gl.host } 42 | func (gl *GitLabURL) GetProjectName() string { return gl.project } 43 | func (gl *GitLabURL) GetBranchName() string { return gl.branch } 44 | func (gl *GitLabURL) GetOwnerName() string { return gl.owner } 45 | func (gl *GitLabURL) GetRepoName() string { return gl.repo } 46 | func (gl *GitLabURL) GetPath() string { return gl.path } 47 | func (gl *GitLabURL) GetToken() string { return gl.token } 48 | func (gl *GitLabURL) GetHttpCloneURL() string { 49 | return fmt.Sprintf("https://%s/%s/%s.git", gl.host, gl.owner, gl.repo) 50 | } 51 | 52 | func (gl *GitLabURL) SetOwnerName(o string) { gl.owner = o } 53 | func (gl *GitLabURL) SetProjectName(project string) { gl.project = project } 54 | func (gl *GitLabURL) SetRepoName(r string) { gl.repo = r } 55 | func (gl *GitLabURL) SetBranchName(branch string) { gl.branch = branch } 56 | func (gl *GitLabURL) SetPath(p string) { gl.path = p } 57 | func (gl *GitLabURL) SetToken(token string) { gl.token = token } 58 | 59 | // Parse URL 60 | func (gl *GitLabURL) Parse(fullURL string) error { 61 | parsedURL, err := giturl.Parse(fullURL) 62 | if err != nil { 63 | return err 64 | } 65 | 66 | gl.host = parsedURL.Host 67 | 68 | index := 0 69 | 70 | splittedRepo := strings.FieldsFunc(parsedURL.Path, func(c rune) bool { return c == '/' }) // trim empty fields from returned slice 71 | if len(splittedRepo) < 1 { 72 | return fmt.Errorf("expecting / or in url path, received: '%s'", parsedURL.Path) 73 | } 74 | 75 | // in gitlab / are separated from blob/tree/raw with a - 76 | for i := range splittedRepo { 77 | if splittedRepo[i] == "-" { 78 | break 79 | } 80 | index = i 81 | } 82 | 83 | gl.owner = strings.Join(splittedRepo[:index], "/") 84 | gl.repo = strings.TrimSuffix(splittedRepo[index], ".git") 85 | index++ 86 | 87 | // root of repo 88 | if len(splittedRepo) < index+1 { 89 | return nil 90 | } 91 | 92 | if splittedRepo[index] == "-" { 93 | index++ // skip "-" symbol in URL 94 | } 95 | 96 | // is file or dir 97 | switch splittedRepo[index] { 98 | case "blob", "tree", "raw": 99 | index++ 100 | } 101 | 102 | if len(splittedRepo) < index+1 { 103 | return nil 104 | } 105 | 106 | gl.branch = splittedRepo[index] 107 | index += 1 108 | 109 | if len(splittedRepo) < index+1 { 110 | return nil 111 | } 112 | gl.path = strings.Join(splittedRepo[index:], "/") 113 | 114 | return nil 115 | } 116 | 117 | // Set the default brach of the repo 118 | func (gl *GitLabURL) SetDefaultBranchName() error { 119 | branch, err := gl.gitLabAPI.GetDefaultBranchName(gl.GetOwnerName(), gl.GetRepoName(), gl.headers()) 120 | if err != nil { 121 | return err 122 | } 123 | gl.branch = branch 124 | return nil 125 | } 126 | 127 | func (gl *GitLabURL) headers() *gitlabapi.Headers { 128 | return &gitlabapi.Headers{Token: gl.GetToken()} 129 | } 130 | -------------------------------------------------------------------------------- /bitbucketparser/v1/parser.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "os" 7 | "strings" 8 | 9 | giturl "github.com/chainguard-dev/git-urls" 10 | "github.com/kubescape/go-git-url/apis" 11 | "github.com/kubescape/go-git-url/apis/bitbucketapi" 12 | ) 13 | 14 | const HOST = "bitbucket.org" 15 | 16 | // NewBitBucketParser empty instance of a bitbucket parser 17 | func NewBitBucketParser() *BitBucketURL { 18 | 19 | return &BitBucketURL{ 20 | bitBucketAPI: bitbucketapi.NewBitBucketAPI(), 21 | host: HOST, 22 | token: os.Getenv("BITBUCKET_TOKEN"), 23 | } 24 | } 25 | 26 | // NewBitBucketParserWithURL parsed instance of a bitbucket parser 27 | func NewBitBucketParserWithURL(fullURL string) (*BitBucketURL, error) { 28 | gl := NewBitBucketParser() 29 | 30 | if err := gl.Parse(fullURL); err != nil { 31 | return gl, err 32 | } 33 | 34 | return gl, nil 35 | } 36 | 37 | func (gl *BitBucketURL) GetURL() *url.URL { 38 | return &url.URL{ 39 | Scheme: "https", 40 | Host: gl.GetHostName(), 41 | Path: fmt.Sprintf("%s/%s", gl.GetOwnerName(), gl.GetRepoName()), 42 | } 43 | } 44 | 45 | func IsHostBitBucket(host string) bool { return strings.HasSuffix(host, HOST) } 46 | 47 | func (gl *BitBucketURL) GetProvider() string { return apis.ProviderBitBucket.String() } 48 | func (gl *BitBucketURL) GetHostName() string { return gl.host } 49 | func (gl *BitBucketURL) GetProjectName() string { return gl.project } 50 | func (gl *BitBucketURL) GetBranchName() string { return gl.branch } 51 | func (gl *BitBucketURL) GetOwnerName() string { return gl.owner } 52 | func (gl *BitBucketURL) GetRepoName() string { return gl.repo } 53 | func (gl *BitBucketURL) GetPath() string { return gl.path } 54 | func (gl *BitBucketURL) GetToken() string { return gl.token } 55 | func (gl *BitBucketURL) GetHttpCloneURL() string { 56 | return fmt.Sprintf("https://bitbucket.org/%s/%s.git", gl.GetOwnerName(), gl.GetRepoName()) 57 | } 58 | 59 | func (gl *BitBucketURL) SetOwnerName(o string) { gl.owner = o } 60 | func (gl *BitBucketURL) SetProjectName(project string) { gl.project = project } 61 | func (gl *BitBucketURL) SetRepoName(r string) { gl.repo = r } 62 | func (gl *BitBucketURL) SetBranchName(branch string) { gl.branch = branch } 63 | func (gl *BitBucketURL) SetPath(p string) { gl.path = p } 64 | func (gl *BitBucketURL) SetToken(token string) { gl.token = token } 65 | 66 | // Parse URL 67 | func (gl *BitBucketURL) Parse(fullURL string) error { 68 | parsedURL, err := giturl.Parse(fullURL) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | index := 0 74 | 75 | splitRepo := strings.FieldsFunc(parsedURL.Path, func(c rune) bool { return c == '/' }) // trim empty fields from returned slice 76 | if len(splitRepo) < 2 { 77 | return fmt.Errorf("expecting / in url path, received: '%s'", parsedURL.Path) 78 | } 79 | gl.owner = splitRepo[index] 80 | index++ 81 | gl.repo = strings.TrimSuffix(splitRepo[index], ".git") 82 | index++ 83 | 84 | // root of repo 85 | if len(splitRepo) < index+1 { 86 | return nil 87 | } 88 | 89 | if splitRepo[index] == "-" { 90 | index++ // skip "-" symbol in URL 91 | } 92 | 93 | // is file or dir 94 | switch splitRepo[index] { 95 | case "src", "raw": 96 | index++ 97 | } 98 | 99 | if len(splitRepo) < index+1 { 100 | return nil 101 | } 102 | 103 | gl.branch = splitRepo[index] 104 | index += 1 105 | 106 | if len(splitRepo) < index+1 { 107 | return nil 108 | } 109 | gl.path = strings.Join(splitRepo[index:], "/") 110 | 111 | return nil 112 | } 113 | 114 | // SetDefaultBranchName sets the default brach of the repo 115 | func (gl *BitBucketURL) SetDefaultBranchName() error { 116 | branch, err := gl.bitBucketAPI.GetDefaultBranchName(gl.GetOwnerName(), gl.GetRepoName(), gl.headers()) 117 | if err != nil { 118 | return err 119 | } 120 | gl.branch = branch 121 | return nil 122 | } 123 | 124 | func (gl *BitBucketURL) headers() *bitbucketapi.Headers { 125 | return &bitbucketapi.Headers{Token: gl.GetToken()} 126 | } 127 | -------------------------------------------------------------------------------- /apis/githubapi/apis.go: -------------------------------------------------------------------------------- 1 | package githubapi 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/kubescape/go-git-url/apis" 9 | ) 10 | 11 | const ( 12 | DEFAULT_HOST string = "github.com" 13 | RAW_HOST string = "raw.githubusercontent.com" 14 | SUBDOMAIN_HOST string = "www.github.com" 15 | ) 16 | 17 | type IGitHubAPI interface { 18 | GetRepoTree(owner, repo, branch string, headers *Headers) (*Tree, error) 19 | GetDefaultBranchName(owner, repo string, headers *Headers) (string, error) 20 | GetLatestCommit(owner, repo, branch string, headers *Headers) (*Commit, error) 21 | // GetCommits(owner, repo, branch string, headers *Headers) ([]Commit, error) 22 | // GetLatestPathCommit(owner, repo, branch, fullPath string, headers *Headers) ([]]Commit, error) 23 | } 24 | type GitHubAPI struct { 25 | httpClient *http.Client 26 | } 27 | 28 | func NewGitHubAPI() *GitHubAPI { return &GitHubAPI{httpClient: &http.Client{}} } 29 | 30 | func (gh *GitHubAPI) GetRepoTree(owner, repo, branch string, headers *Headers) (*Tree, error) { 31 | treeAPI := APIRepoTree(owner, repo, branch) 32 | body, err := apis.HttpGet(gh.httpClient, treeAPI, headers.ToMap()) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | var tree Tree 38 | if err = json.Unmarshal([]byte(body), &tree); err != nil { 39 | return nil, fmt.Errorf("failed to unmarshal response body from '%s', reason: %s", treeAPI, err.Error()) 40 | } 41 | return &tree, nil 42 | 43 | } 44 | 45 | func (gh *GitHubAPI) GetDefaultBranchName(owner, repo string, headers *Headers) (string, error) { 46 | 47 | body, err := apis.HttpGet(gh.httpClient, APIMetadata(owner, repo), headers.ToMap()) 48 | if err != nil { 49 | return "", err 50 | } 51 | 52 | var data githubDefaultBranchAPI 53 | err = json.Unmarshal([]byte(body), &data) 54 | if err != nil { 55 | return "", err 56 | } 57 | return data.DefaultBranch, nil 58 | } 59 | 60 | func (gh *GitHubAPI) GetLatestCommit(owner, repo, branch string, headers *Headers) (*Commit, error) { 61 | 62 | body, err := apis.HttpGet(gh.httpClient, APILastCommitsOfBranch(owner, repo, branch), headers.ToMap()) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | var data Commit 68 | err = json.Unmarshal([]byte(body), &data) 69 | if err != nil { 70 | return &data, err 71 | } 72 | return &data, nil 73 | } 74 | 75 | // Get latest commit data of path/file 76 | func (gh *GitHubAPI) GetFileLatestCommit(owner, repo, branch, fullPath string, headers *Headers) ([]Commit, error) { 77 | 78 | body, err := apis.HttpGet(gh.httpClient, APILastCommitsOfPath(owner, repo, branch, fullPath), headers.ToMap()) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | var data []Commit 84 | err = json.Unmarshal([]byte(body), &data) 85 | if err != nil { 86 | return data, err 87 | } 88 | return data, nil 89 | } 90 | 91 | // APIRepoTree github tree api 92 | func APIRepoTree(owner, repo, branch string) string { 93 | return fmt.Sprintf("https://api.%s/repos/%s/%s/git/trees/%s?recursive=1", DEFAULT_HOST, owner, repo, branch) 94 | } 95 | 96 | // APIRaw github raw file api 97 | func APIRaw(owner, repo, branch, path string) string { 98 | return fmt.Sprintf("https://%s/%s/%s/%s/%s", RAW_HOST, owner, repo, branch, path) 99 | } 100 | 101 | // APIDefaultBranch github repo metadata api 102 | func APIMetadata(owner, repo string) string { 103 | return fmt.Sprintf("https://api.%s/repos/%s/%s", DEFAULT_HOST, owner, repo) 104 | } 105 | 106 | // APILastCommits github last commit api 107 | func APILastCommits(owner, repo string) string { 108 | return fmt.Sprintf("https://api.%s/repos/%s/%s/commits", DEFAULT_HOST, owner, repo) 109 | } 110 | 111 | // APILastCommitsOfBranch github last commit of specific branch api 112 | func APILastCommitsOfBranch(owner, repo, branch string) string { 113 | return fmt.Sprintf("%s/%s", APILastCommits(owner, repo), branch) 114 | } 115 | 116 | // APILastCommitsOfPath github last commit of specific branch api 117 | func APILastCommitsOfPath(owner, repo, branch, path string) string { 118 | return fmt.Sprintf("%s?path=%s", APILastCommitsOfBranch(owner, repo, branch), path) 119 | } 120 | -------------------------------------------------------------------------------- /githubparser/v1/parser_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | var ( 11 | urlA = "https://github.com/kubescape/go-git-url" 12 | urlB = "https://github.com/kubescape/go-git-url/blob/master/files/file0.json" 13 | urlC = "https://github.com/kubescape/go-git-url/tree/master/files" 14 | urlD = "https://raw.githubusercontent.com/kubescape/go-git-url/master/files/file0.json" 15 | urlE = "git@github.com:kubescape/go-git-url.git" 16 | urlF = "git@github.com:foobar/kubescape/go-git-url.git" 17 | urlG = "https://www.github.com/kubescape/go-git-url" 18 | ) 19 | 20 | func TestNewGitHubParserWithURL(t *testing.T) { 21 | { 22 | gh, err := NewGitHubParserWithURL(urlA) 23 | assert.NoError(t, err) 24 | assert.Equal(t, "github.com", gh.GetHostName()) 25 | assert.Equal(t, "github", gh.GetProvider()) 26 | assert.Equal(t, "kubescape", gh.GetOwnerName()) 27 | assert.Equal(t, "go-git-url", gh.GetRepoName()) 28 | assert.Equal(t, urlA, gh.GetURL().String()) 29 | assert.Equal(t, "", gh.GetBranchName()) 30 | assert.Equal(t, "", gh.GetPath()) 31 | } 32 | { 33 | gh, err := NewGitHubParserWithURL(urlB) 34 | assert.NoError(t, err) 35 | assert.Equal(t, "github.com", gh.GetHostName()) 36 | assert.Equal(t, "kubescape", gh.GetOwnerName()) 37 | assert.Equal(t, "go-git-url", gh.GetRepoName()) 38 | assert.Equal(t, "master", gh.GetBranchName()) 39 | assert.Equal(t, "files/file0.json", gh.GetPath()) 40 | assert.Equal(t, urlA, gh.GetURL().String()) 41 | } 42 | { 43 | gh, err := NewGitHubParserWithURL(urlC) 44 | assert.NoError(t, err) 45 | assert.Equal(t, "github.com", gh.GetHostName()) 46 | assert.Equal(t, "kubescape", gh.GetOwnerName()) 47 | assert.Equal(t, "go-git-url", gh.GetRepoName()) 48 | assert.Equal(t, "master", gh.GetBranchName()) 49 | assert.Equal(t, "files", gh.GetPath()) 50 | assert.Equal(t, urlA, gh.GetURL().String()) 51 | } 52 | { 53 | gh, err := NewGitHubParserWithURL(urlD) 54 | assert.NoError(t, err) 55 | assert.Equal(t, "github.com", gh.GetHostName()) 56 | assert.Equal(t, "kubescape", gh.GetOwnerName()) 57 | assert.Equal(t, "go-git-url", gh.GetRepoName()) 58 | assert.Equal(t, "master", gh.GetBranchName()) 59 | assert.Equal(t, "files/file0.json", gh.GetPath()) 60 | assert.Equal(t, urlA, gh.GetURL().String()) 61 | } 62 | { 63 | gh, err := NewGitHubParserWithURL(urlE) 64 | assert.NoError(t, err) 65 | assert.Equal(t, "github.com", gh.GetHostName()) 66 | assert.Equal(t, "kubescape", gh.GetOwnerName()) 67 | assert.Equal(t, "go-git-url", gh.GetRepoName()) 68 | assert.Equal(t, urlA, gh.GetURL().String()) 69 | assert.Equal(t, "", gh.GetBranchName()) 70 | assert.Equal(t, "", gh.GetPath()) 71 | } 72 | { 73 | assert.NotPanics(t, func() { 74 | _, _ = NewGitHubParserWithURL(urlF) 75 | }) 76 | } 77 | { 78 | gh, err := NewGitHubParserWithURL(urlG) 79 | assert.NoError(t, err) 80 | assert.Equal(t, "github.com", gh.GetHostName()) 81 | assert.Equal(t, "kubescape", gh.GetOwnerName()) 82 | assert.Equal(t, "go-git-url", gh.GetRepoName()) 83 | assert.Equal(t, urlA, gh.GetURL().String()) 84 | assert.Equal(t, "", gh.GetBranchName()) 85 | assert.Equal(t, "", gh.GetPath()) 86 | } 87 | } 88 | 89 | func TestSetDefaultBranch(t *testing.T) { 90 | { 91 | gh, err := NewGitHubParserWithURL(urlA) 92 | assert.NoError(t, err) 93 | assert.NoError(t, gh.SetDefaultBranchName()) 94 | assert.Equal(t, "master", gh.GetBranchName()) 95 | } 96 | { 97 | gh, err := NewGitHubParserWithURL(strings.ReplaceAll(urlB, "master", "dev")) 98 | assert.NoError(t, err) 99 | assert.NoError(t, gh.SetDefaultBranchName()) 100 | assert.Equal(t, "master", gh.GetBranchName()) 101 | } 102 | { 103 | gh, err := NewGitHubParserWithURL(urlE) 104 | assert.NoError(t, err) 105 | assert.NoError(t, gh.SetDefaultBranchName()) 106 | assert.Equal(t, "master", gh.GetBranchName()) 107 | } 108 | { 109 | gh, err := NewGitHubParserWithURL(urlG) 110 | assert.NoError(t, err) 111 | assert.NoError(t, gh.SetDefaultBranchName()) 112 | assert.Equal(t, "master", gh.GetBranchName()) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /gitlabparser/v1/parser_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestNewGitHubParserWithURL(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | url string 14 | want *GitLabURL 15 | wantErr assert.ErrorAssertionFunc 16 | }{ 17 | { 18 | name: "general case", 19 | url: "https://gitlab.com/kubescape/testing", 20 | want: &GitLabURL{ 21 | host: "gitlab.com", 22 | owner: "kubescape", 23 | repo: "testing", 24 | }, 25 | wantErr: assert.NoError, 26 | }, 27 | { 28 | name: "file", 29 | url: "https://gitlab.com/kubescape/testing/-/blob/main/stable/acs-engine-autoscaler/Chart.yaml", 30 | want: &GitLabURL{ 31 | host: "gitlab.com", 32 | owner: "kubescape", 33 | repo: "testing", 34 | branch: "main", 35 | path: "stable/acs-engine-autoscaler/Chart.yaml", 36 | }, 37 | wantErr: assert.NoError, 38 | }, 39 | { 40 | name: "branch", 41 | url: "https://gitlab.com/kubescape/testing/-/blob/dev/README.md", 42 | want: &GitLabURL{ 43 | host: "gitlab.com", 44 | owner: "kubescape", 45 | repo: "testing", 46 | branch: "dev", 47 | path: "README.md", 48 | }, 49 | wantErr: assert.NoError, 50 | }, 51 | { 52 | name: "branch", 53 | url: "https://gitlab.com/kubescape/testing/-/tree/dev", 54 | want: &GitLabURL{ 55 | host: "gitlab.com", 56 | owner: "kubescape", 57 | repo: "testing", 58 | branch: "dev", 59 | }, 60 | wantErr: assert.NoError, 61 | }, 62 | { 63 | name: "tag", 64 | url: "https://gitlab.com/kubescape/testing/-/blob/v0.0.0/README.md", 65 | want: &GitLabURL{ 66 | host: "gitlab.com", 67 | owner: "kubescape", 68 | repo: "testing", 69 | branch: "v0.0.0", 70 | path: "README.md", 71 | }, 72 | wantErr: assert.NoError, 73 | }, 74 | { 75 | name: "raw", 76 | url: "https://gitlab.com/kubescape/testing/-/raw/main/stable/acs-engine-autoscaler/Chart.yaml", 77 | want: &GitLabURL{ 78 | host: "gitlab.com", 79 | owner: "kubescape", 80 | repo: "testing", 81 | branch: "main", 82 | path: "stable/acs-engine-autoscaler/Chart.yaml", 83 | }, 84 | wantErr: assert.NoError, 85 | }, 86 | { 87 | name: "scp-like syntax", 88 | url: "git@gitlab.com:owner/repo.git", 89 | want: &GitLabURL{ 90 | host: "gitlab.com", 91 | owner: "owner", 92 | repo: "repo", 93 | }, 94 | wantErr: assert.NoError, 95 | }, 96 | { 97 | name: "project and subproject", 98 | url: "https://gitlab.com/matthyx1/subgroup1/project1.git", 99 | want: &GitLabURL{ 100 | host: "gitlab.com", 101 | owner: "matthyx1/subgroup1", 102 | repo: "project1", 103 | }, 104 | wantErr: assert.NoError, 105 | }, 106 | { 107 | name: "project and subproject", 108 | url: "https://gitlab.com/matthyx1/subgroup1/subsubgroup1/project1.git", 109 | want: &GitLabURL{ 110 | host: "gitlab.com", 111 | owner: "matthyx1/subgroup1/subsubgroup1", 112 | repo: "project1", 113 | }, 114 | wantErr: assert.NoError, 115 | }, 116 | { 117 | name: "self-hosted", 118 | url: "https://gitlab.host.com/kubescape/testing", 119 | want: &GitLabURL{ 120 | host: "gitlab.host.com", 121 | owner: "kubescape", 122 | repo: "testing", 123 | }, 124 | wantErr: assert.NoError, 125 | }, 126 | { 127 | name: "repo only", 128 | url: "https://git.host.com/repo.git", 129 | want: &GitLabURL{ 130 | host: "git.host.com", 131 | owner: "", 132 | repo: "repo", 133 | }, 134 | wantErr: assert.NoError, 135 | }, 136 | } 137 | for _, tt := range tests { 138 | t.Run(tt.name, func(t *testing.T) { 139 | got, err := NewGitLabParserWithURL("", tt.url) 140 | got.gitLabAPI = nil 141 | if !tt.wantErr(t, err, fmt.Sprintf("NewGitLabParserWithURL(%v)", tt.url)) { 142 | return 143 | } 144 | assert.Equalf(t, tt.want, got, "NewGitLabParserWithURL(%v)", tt.url) 145 | }) 146 | } 147 | } 148 | 149 | func TestIsHostGitLab(t *testing.T) { 150 | tests := []struct { 151 | name string 152 | want bool 153 | }{ 154 | { 155 | name: "gitlab.com", 156 | want: true, 157 | }, 158 | { 159 | name: "gitlab.host.com", 160 | want: true, 161 | }, 162 | { 163 | name: "github.com", 164 | want: false, 165 | }, 166 | } 167 | for _, tt := range tests { 168 | t.Run(tt.name, func(t *testing.T) { 169 | assert.Equalf(t, tt.want, IsHostGitLab(tt.name), "IsHostGitLab(%v)", tt.name) 170 | }) 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /apis/githubapi/datastructures.go: -------------------------------------------------------------------------------- 1 | package githubapi 2 | 3 | import "time" 4 | 5 | type ObjectType string 6 | 7 | const ( 8 | ObjectTypeDir ObjectType = "tree" 9 | ObjectTypeFile ObjectType = "blob" 10 | ) 11 | 12 | type InnerTree struct { 13 | Path string `json:"path"` 14 | Mode string `json:"mode"` 15 | SHA string `json:"sha"` 16 | URL string `json:"url"` 17 | Type ObjectType `json:"type"` 18 | } 19 | type Tree struct { 20 | InnerTrees []InnerTree `json:"tree"` 21 | SHA string `json:"sha"` 22 | URL string `json:"url"` 23 | Truncated bool `json:"truncated"` 24 | } 25 | 26 | type githubDefaultBranchAPI struct { 27 | DefaultBranch string `json:"default_branch"` 28 | } 29 | 30 | type Headers struct { 31 | Token string 32 | } 33 | 34 | // LatestCommit returned structure 35 | type Commit struct { 36 | SHA string `json:"sha"` 37 | NodeID string `json:"node_id"` 38 | Commit CommitsMetadata `json:"commit"` 39 | URL string `json:"url"` 40 | HtmlURL string `json:"html_url"` 41 | CommentsURL string `json:"comments_url"` 42 | Author Author `json:"author"` 43 | Committer Committer `json:"committer"` 44 | Parents []struct { 45 | SHA string `json:"sha"` 46 | URL string `json:"url"` 47 | HtmlURL string `json:"html_url"` 48 | } `json:"parents"` 49 | Stats struct { 50 | Total int `json:"total"` 51 | Additions int `json:"additions"` 52 | Deletions int `json:"deletions"` 53 | } `json:"stats"` 54 | Files []Files `json:"files"` 55 | } 56 | 57 | type Files struct { 58 | SHA string `json:"sha"` 59 | Filename string `json:"filename"` 60 | Status string `json:"status"` 61 | Additions int `json:"additions"` 62 | Deletions int `json:"deletions"` 63 | Changes int `json:"changes"` 64 | BlobURL string `json:"blob_url"` 65 | RawURL string `json:"raw_url"` 66 | ContentsURL string `json:"contents_url"` 67 | Patch string `json:"patch"` 68 | } 69 | type CommitsMetadata struct { 70 | Author struct { 71 | Name string `json:"name"` 72 | Email string `json:"email"` 73 | Date time.Time `json:"date"` 74 | } `json:"author"` 75 | Committer struct { 76 | Name string `json:"name"` 77 | Email string `json:"email"` 78 | Date time.Time `json:"date"` 79 | } `json:"committer"` 80 | Message string `json:"message"` 81 | Tree struct { 82 | SHA string `json:"sha"` 83 | URL string `json:"url"` 84 | } `json:"tree"` 85 | URL string `json:"url"` 86 | CommentCount int `json:"comment_count"` 87 | Verification struct { 88 | Verified bool `json:"verified"` 89 | Reason string `json:"reason"` 90 | Signature interface{} `json:"signature"` 91 | Payload interface{} `json:"payload"` 92 | } `json:"verification"` 93 | } 94 | type Author struct { 95 | Login string `json:"login"` 96 | ID int `json:"id"` 97 | NodeID string `json:"node_id"` 98 | AvatarURL string `json:"avatar_url"` 99 | GravatarID string `json:"gravatar_id"` 100 | URL string `json:"url"` 101 | HtmlURL string `json:"html_url"` 102 | FollowersURL string `json:"followers_url"` 103 | FollowingURL string `json:"following_url"` 104 | GistsURL string `json:"gists_url"` 105 | StarredURL string `json:"starred_url"` 106 | SubscriptionsURL string `json:"subscriptions_url"` 107 | OrganizationsURL string `json:"organizations_url"` 108 | ReposURL string `json:"repos_url"` 109 | EventsURL string `json:"events_url"` 110 | ReceivedEventsURL string `json:"received_events_url"` 111 | Type string `json:"type"` 112 | SiteAdmin bool `json:"site_admin"` 113 | } 114 | 115 | type Committer struct { 116 | Login string `json:"login"` 117 | ID int `json:"id"` 118 | NodeID string `json:"node_id"` 119 | AvatarURL string `json:"avatar_url"` 120 | GravatarID string `json:"gravatar_id"` 121 | URL string `json:"url"` 122 | HtmlURL string `json:"html_url"` 123 | FollowersURL string `json:"followers_url"` 124 | FollowingURL string `json:"following_url"` 125 | GistsURL string `json:"gists_url"` 126 | StarredURL string `json:"starred_url"` 127 | SubscriptionsURL string `json:"subscriptions_url"` 128 | OrganizationsURL string `json:"organizations_url"` 129 | ReposURL string `json:"repos_url"` 130 | EventsURL string `json:"events_url"` 131 | ReceivedEventsURL string `json:"received_events_url"` 132 | Type string `json:"type"` 133 | SiteAdmin bool `json:"site_admin"` 134 | } 135 | -------------------------------------------------------------------------------- /azureparser/v1/parser.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "os" 7 | "strings" 8 | 9 | giturl "github.com/chainguard-dev/git-urls" 10 | "github.com/kubescape/go-git-url/apis" 11 | "github.com/kubescape/go-git-url/apis/azureapi" 12 | ) 13 | 14 | const HOST = "azure.com" 15 | const HOST_DEV = "dev.azure.com" 16 | const HOST_PROD = "prod.azure.com" 17 | 18 | // NewAzureParser empty instance of a azure parser 19 | func NewAzureParser() *AzureURL { 20 | 21 | return &AzureURL{ 22 | azureAPI: azureapi.NewAzureAPI(), 23 | host: HOST, 24 | token: os.Getenv("AZURE_TOKEN"), 25 | } 26 | } 27 | 28 | // NewAzureParserWithURL parsed instance of a azure parser 29 | func NewAzureParserWithURL(fullURL string) (*AzureURL, error) { 30 | az := NewAzureParser() 31 | 32 | if err := az.Parse(fullURL); err != nil { 33 | return az, err 34 | } 35 | 36 | return az, nil 37 | } 38 | 39 | func (az *AzureURL) GetURL() *url.URL { 40 | return &url.URL{ 41 | Scheme: "https", 42 | Host: az.host, 43 | Path: fmt.Sprintf("%s/%s/_git/%s", az.GetOwnerName(), az.GetProjectName(), az.GetRepoName()), 44 | } 45 | } 46 | 47 | func IsHostAzure(host string) bool { return strings.HasSuffix(host, HOST) } 48 | 49 | func (az *AzureURL) GetProvider() string { return apis.ProviderAzure.String() } 50 | func (az *AzureURL) GetHostName() string { return az.host } 51 | func (az *AzureURL) GetProjectName() string { return az.project } 52 | func (az *AzureURL) GetBranchName() string { return az.branch } 53 | func (az *AzureURL) GetTag() string { return az.tag } 54 | func (az *AzureURL) GetOwnerName() string { return az.owner } 55 | func (az *AzureURL) GetRepoName() string { return az.repo } 56 | func (az *AzureURL) GetPath() string { return az.path } 57 | func (az *AzureURL) GetToken() string { return az.token } 58 | func (az *AzureURL) GetHttpCloneURL() string { 59 | return fmt.Sprintf("https://%s/%s/%s/_git/%s", az.GetHostName(), az.GetOwnerName(), az.GetProjectName(), az.GetRepoName()) 60 | } 61 | 62 | func (az *AzureURL) SetOwnerName(o string) { az.owner = o } 63 | func (az *AzureURL) SetProjectName(project string) { az.project = project } 64 | func (az *AzureURL) SetRepoName(r string) { az.repo = r } 65 | func (az *AzureURL) SetBranchName(branch string) { az.branch = branch } 66 | func (az *AzureURL) SetTag(tag string) { az.tag = tag } 67 | func (az *AzureURL) SetPath(p string) { az.path = p } 68 | func (az *AzureURL) SetToken(token string) { az.token = token } 69 | 70 | // Parse URL 71 | func (az *AzureURL) Parse(fullURL string) error { 72 | parsedURL, err := giturl.Parse(fullURL) 73 | if err != nil { 74 | return err 75 | } 76 | az.host = parsedURL.Host 77 | 78 | if strings.HasPrefix(az.host, "ssh") { 79 | az.host = strings.TrimPrefix(az.host, "ssh.") 80 | return az.parseHostSSH(parsedURL) 81 | } 82 | return az.parseHostHTTP(parsedURL) 83 | } 84 | 85 | func (az *AzureURL) parseHostSSH(parsedURL *url.URL) error { 86 | splittedRepo := strings.FieldsFunc(parsedURL.Path, func(c rune) bool { return c == '/' }) // trim empty fields from returned slice 87 | 88 | if len(splittedRepo) < 3 || len(splittedRepo) > 5 { 89 | return fmt.Errorf("expecting v/// in url path, received: '%s'", parsedURL.Path) 90 | } 91 | 92 | index := 0 93 | if len(splittedRepo) == 4 { 94 | index++ 95 | } 96 | az.owner = splittedRepo[index] 97 | az.project = splittedRepo[index+1] 98 | az.repo = splittedRepo[index+2] 99 | 100 | return nil 101 | } 102 | func (az *AzureURL) parseHostHTTP(parsedURL *url.URL) error { 103 | splittedRepo := strings.FieldsFunc(parsedURL.Path, func(c rune) bool { return c == '/' }) // trim empty fields from returned slice 104 | if len(splittedRepo) < 4 || splittedRepo[2] != "_git" { 105 | return fmt.Errorf("expecting //_git/ in url path, received: '%s'", parsedURL.Path) 106 | } 107 | az.owner = splittedRepo[0] 108 | az.project = splittedRepo[1] 109 | az.repo = splittedRepo[3] 110 | 111 | if v := parsedURL.Query().Get("version"); v != "" { 112 | if strings.HasPrefix(v, "GB") { 113 | az.branch = strings.TrimPrefix(v, "GB") 114 | } 115 | if strings.HasPrefix(v, "GT") { 116 | az.tag = strings.TrimPrefix(v, "GT") 117 | } 118 | } 119 | 120 | if v := parsedURL.Query().Get("path"); v != "" { 121 | az.path = v 122 | } 123 | 124 | return nil 125 | } 126 | 127 | // Set the default brach of the repo 128 | func (az *AzureURL) SetDefaultBranchName() error { 129 | 130 | branch, err := az.azureAPI.GetDefaultBranchName(az.GetOwnerName(), az.GetProjectName(), az.GetRepoName(), az.headers()) 131 | 132 | if err != nil { 133 | return err 134 | } 135 | az.branch = branch 136 | return nil 137 | } 138 | 139 | func (az *AzureURL) headers() *azureapi.Headers { 140 | return &azureapi.Headers{Token: az.GetToken()} 141 | } 142 | -------------------------------------------------------------------------------- /apis/gitlabapi/apis.go: -------------------------------------------------------------------------------- 1 | package gitlabapi 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "strings" 9 | 10 | "github.com/kubescape/go-git-url/apis" 11 | ) 12 | 13 | type IGitLabAPI interface { 14 | GetRepoTree(owner, repo, branch string, headers *Headers) (*Tree, error) 15 | GetDefaultBranchName(owner, repo string, headers *Headers) (string, error) 16 | GetLatestCommit(owner, repo, branch string, headers *Headers) (*Commit, error) 17 | } 18 | 19 | type GitLabAPI struct { 20 | host string 21 | httpClient *http.Client 22 | } 23 | 24 | func NewGitLabAPI(host string) *GitLabAPI { 25 | return &GitLabAPI{ 26 | host: host, 27 | httpClient: &http.Client{}, 28 | } 29 | } 30 | 31 | func getProjectId(owner, repo string) string { 32 | return strings.ReplaceAll(owner+"/"+repo, "/", "%2F") 33 | } 34 | 35 | func (gl *GitLabAPI) GetRepoTree(owner, repo, branch string, headers *Headers) (*Tree, error) { 36 | id := getProjectId(owner, repo) 37 | 38 | treeAPI := APIRepoTree(gl.host, id, branch) 39 | body, err := apis.HttpGet(gl.httpClient, treeAPI, headers.ToMap()) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | var tree Tree 45 | if err = json.Unmarshal([]byte(body), &tree); err != nil { 46 | return nil, fmt.Errorf("failed to unmarshal response body from '%s', reason: %s", treeAPI, err.Error()) 47 | } 48 | 49 | return &tree, nil 50 | 51 | } 52 | 53 | func (gl *GitLabAPI) GetDefaultBranchName(owner, repo string, headers *Headers) (string, error) { 54 | id := getProjectId(owner, repo) 55 | 56 | body, err := apis.HttpGet(gl.httpClient, APIMetadata(gl.host, id), headers.ToMap()) 57 | if err != nil { 58 | return "", err 59 | } 60 | 61 | var data gitlabDefaultBranchAPI 62 | if err = json.Unmarshal([]byte(body), &data); err != nil { 63 | return "", err 64 | } 65 | 66 | for i := range data { 67 | if data[i].DefaultBranch { 68 | return data[i].Name, nil 69 | } 70 | } 71 | 72 | errAPI := errors.New("unable to find default branch from the GitLab API") 73 | return "", errAPI 74 | } 75 | 76 | func (gl *GitLabAPI) GetLatestCommit(owner, repo, branch string, headers *Headers) (*Commit, error) { 77 | id := getProjectId(owner, repo) 78 | 79 | body, err := apis.HttpGet(gl.httpClient, APILastCommitsOfBranch(gl.host, id, branch), headers.ToMap()) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | var data []Commit 85 | err = json.Unmarshal([]byte(body), &data) 86 | if err != nil { 87 | return &data[0], err 88 | } 89 | return &data[0], nil 90 | } 91 | 92 | // APIRepoTree GitLab tree api 93 | // API Ref: https://docs.gitlab.com/ee/api/repositories.html 94 | func APIRepoTree(host, id, branch string) string { 95 | return fmt.Sprintf("https://%s/api/v4/projects/%s/repository/tree?ref=%s", host, id, branch) 96 | } 97 | 98 | // APIRaw GitLab : Get raw file from repository 99 | // API Ref: https://docs.gitlab.com/ee/api/repository_files.html#get-raw-file-from-repository 100 | // Example: https://gitlab.com/api/v4/projects/23383112/repository/files/app%2Findex.html/raw 101 | func APIRaw(host, owner, repo, path string) string { 102 | id := getProjectId(owner, repo) 103 | 104 | return fmt.Sprintf("https://%s/api/v4/projects/%s/repository/files/%s/raw", host, id, path) 105 | } 106 | 107 | // APIDefaultBranch GitLab : Branch metadata; list repo branches 108 | // API Ref: https://docs.gitlab.com/ee/api/branches.html#list-repository-branches 109 | // Example: https://gitlab.com/api/v4/projects/nanuchi%2Fdeveloping-with-docker/repository/branches 110 | func APIMetadata(host, id string) string { 111 | return fmt.Sprintf("https://%s/api/v4/projects/%s/repository/branches", host, id) 112 | } 113 | 114 | // APILastCommits GitLab : List repository commits 115 | // API Ref: https://docs.gitlab.com/ee/api/commits.html#list-repository-commits 116 | // Example: https://gitlab.com/api/v4/projects/nanuchi%2Fdeveloping-with-docker/repository/commits 117 | func APILastCommits(host, id string) string { 118 | return fmt.Sprintf("https://%s/api/v4/projects/%s/repository/commits", host, id) 119 | } 120 | 121 | // TODO: These return list of commits, and we might want just a single latest commit. Pls check with this later: 122 | 123 | // APILastCommitsOfBranch GitLab : Last commits of specific branch 124 | // API Ref: https://docs.gitlab.com/ee/api/commits.html#list-repository-commits 125 | // Example: https://gitlab.com/api/v4/projects/nanuchi%2Fdeveloping-with-docker/repository/commits?ref_name=feature/k8s-in-hour 126 | func APILastCommitsOfBranch(host, id, branch string) string { 127 | return fmt.Sprintf("%s?ref_name=%s", APILastCommits(host, id), branch) 128 | } 129 | 130 | // APILastCommitsOfPath GitLab : Last commits of specific branch & specified path 131 | // API Ref: https://docs.gitlab.com/ee/api/commits.html#list-repository-commits 132 | // Example: https://gitlab.com/api/v4/projects/nanuchi%2Fdeveloping-with-docker/repository/commits?ref_name=master&path=app/server.js 133 | func APILastCommitsOfPath(host, id, branch, path string) string { 134 | return fmt.Sprintf("%s&path=%s", APILastCommitsOfBranch(host, id, branch), path) 135 | } 136 | -------------------------------------------------------------------------------- /azureparser/v1/parser_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | var ( 10 | urlA = "https://dev.azure.com/dwertent/ks-testing-public/_git/ks-testing-public" 11 | urlB = "https://dev.azure.com/dwertent/ks-testing-public/_git/ks-testing-public?path=/rules-tests/alert-rw-hostpath/deployment/expected.json" 12 | urlC = "https://dev.azure.com/dwertent/ks-testing-public/_git/ks-testing-public?path=/scripts&version=GBdev&_a=contents" 13 | urlD = "https://dev.azure.com/dwertent/ks-testing-public/_git/ks-testing-public?path=/scripts&version=GTv1.0.1&_a=contents" 14 | urlE = "https://dev.azure.com/dwertent/ks-testing-public/_git/ks-testing-public?path=%2F&version=GBdev" 15 | urlF = "https://dwertent@dev.azure.com/dwertent/ks-testing-public/_git/ks-testing-public" 16 | // scp-like syntax supported by git for ssh 17 | // see: https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS 18 | // regular form 19 | urlG = "git@ssh.dev.azure.com:v3/dwertent/ks-testing-public/ks-testing-public" 20 | ) 21 | 22 | func TestNewAzureParserWithURL(t *testing.T) { 23 | { 24 | az, err := NewAzureParserWithURL(urlA) 25 | assert.NoError(t, err) 26 | assert.Equal(t, "dev.azure.com", az.GetHostName()) 27 | assert.Equal(t, "azure", az.GetProvider()) 28 | assert.Equal(t, "dwertent", az.GetOwnerName()) 29 | assert.Equal(t, "ks-testing-public", az.GetRepoName()) 30 | assert.Equal(t, urlA, az.GetURL().String()) 31 | assert.Equal(t, "", az.GetBranchName()) 32 | assert.Equal(t, "", az.GetPath()) 33 | } 34 | { 35 | az, err := NewAzureParserWithURL(urlB) 36 | assert.NoError(t, err) 37 | assert.Equal(t, "dev.azure.com", az.GetHostName()) 38 | assert.Equal(t, "azure", az.GetProvider()) 39 | assert.Equal(t, "dwertent", az.GetOwnerName()) 40 | assert.Equal(t, "ks-testing-public", az.GetRepoName()) 41 | assert.Equal(t, urlA, az.GetURL().String()) 42 | assert.Equal(t, "", az.GetBranchName()) 43 | assert.Equal(t, "/rules-tests/alert-rw-hostpath/deployment/expected.json", az.GetPath()) 44 | } 45 | { 46 | az, err := NewAzureParserWithURL(urlC) 47 | assert.NoError(t, err) 48 | assert.Equal(t, "dev.azure.com", az.GetHostName()) 49 | assert.Equal(t, "azure", az.GetProvider()) 50 | assert.Equal(t, "dwertent", az.GetOwnerName()) 51 | assert.Equal(t, "ks-testing-public", az.GetRepoName()) 52 | assert.Equal(t, urlA, az.GetURL().String()) 53 | assert.Equal(t, "dev", az.GetBranchName()) 54 | assert.Equal(t, "", az.GetTag()) 55 | assert.Equal(t, "/scripts", az.GetPath()) 56 | } 57 | { 58 | az, err := NewAzureParserWithURL(urlD) 59 | assert.NoError(t, err) 60 | assert.Equal(t, "dev.azure.com", az.GetHostName()) 61 | assert.Equal(t, "azure", az.GetProvider()) 62 | assert.Equal(t, "dwertent", az.GetOwnerName()) 63 | assert.Equal(t, "ks-testing-public", az.GetRepoName()) 64 | assert.Equal(t, urlA, az.GetURL().String()) 65 | assert.Equal(t, "v1.0.1", az.GetTag()) 66 | assert.Equal(t, "", az.GetBranchName()) 67 | assert.Equal(t, "/scripts", az.GetPath()) 68 | } 69 | { 70 | az, err := NewAzureParserWithURL(urlE) 71 | assert.NoError(t, err) 72 | assert.Equal(t, "dev.azure.com", az.GetHostName()) 73 | assert.Equal(t, "azure", az.GetProvider()) 74 | assert.Equal(t, "dwertent", az.GetOwnerName()) 75 | assert.Equal(t, "ks-testing-public", az.GetRepoName()) 76 | assert.Equal(t, urlA, az.GetURL().String()) 77 | assert.Equal(t, "", az.GetTag()) 78 | assert.Equal(t, "dev", az.GetBranchName()) 79 | assert.Equal(t, "/", az.GetPath()) 80 | } 81 | { 82 | az, err := NewAzureParserWithURL(urlF) 83 | assert.NoError(t, err) 84 | assert.Equal(t, "dev.azure.com", az.GetHostName()) 85 | assert.Equal(t, "azure", az.GetProvider()) 86 | assert.Equal(t, "dwertent", az.GetOwnerName()) 87 | assert.Equal(t, "ks-testing-public", az.GetRepoName()) 88 | assert.Equal(t, urlA, az.GetURL().String()) 89 | assert.Equal(t, "", az.GetBranchName()) 90 | assert.Equal(t, "", az.GetPath()) 91 | } 92 | { 93 | az, err := NewAzureParserWithURL(urlG) 94 | assert.NoError(t, err) 95 | assert.NoError(t, err) 96 | assert.Equal(t, "dev.azure.com", az.GetHostName()) 97 | assert.Equal(t, "azure", az.GetProvider()) 98 | assert.Equal(t, "dwertent", az.GetOwnerName()) 99 | assert.Equal(t, "ks-testing-public", az.GetRepoName()) 100 | assert.Equal(t, urlA, az.GetURL().String()) 101 | assert.Equal(t, "", az.GetBranchName()) 102 | assert.Equal(t, "", az.GetPath()) 103 | } 104 | } 105 | 106 | func TestSetDefaultBranch(t *testing.T) { 107 | { 108 | az, err := NewAzureParserWithURL(urlA) 109 | assert.NoError(t, err) 110 | assert.NoError(t, az.SetDefaultBranchName()) 111 | assert.Equal(t, "master", az.GetBranchName()) 112 | } 113 | { 114 | az, err := NewAzureParserWithURL(urlE) 115 | assert.NoError(t, err) 116 | assert.NoError(t, az.SetDefaultBranchName()) 117 | assert.Equal(t, "master", az.GetBranchName()) 118 | } 119 | { 120 | az, err := NewAzureParserWithURL(urlF) 121 | assert.NoError(t, err) 122 | assert.NoError(t, az.SetDefaultBranchName()) 123 | assert.Equal(t, "master", az.GetBranchName()) 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /bitbucketparser/v1/parser_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | var ( 10 | urlA = "https://bitbucket.org/matthyx/ks-testing-public" // general case 11 | urlB = "https://bitbucket.org/matthyx/ks-testing-public/src/master/rules/etcd-encryption-native/raw.rego" // file 12 | urlC = "https://bitbucket.org/matthyx/ks-testing-public/src/dev/README.md" // branch 13 | urlD = "https://bitbucket.org/matthyx/ks-testing-public/src/dev/" // branch 14 | urlE = "https://bitbucket.org/matthyx/ks-testing-public/src/v1.0.178/README.md" // TODO fix tag 15 | urlF = "https://bitbucket.org/matthyx/ks-testing-public/raw/4502b9b51ee3ac1ea649bacfa0f48ebdeab05f4a/README.md" // TODO fix sha 16 | // scp-like syntax supported by git for ssh 17 | // see: https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS 18 | // regular form 19 | urlG = "git@bitbucket.org:matthyx/ks-testing-public.git" 20 | // unexpected form: should not panic 21 | urlH = "git@bitbucket.org:matthyx/to/ks-testing-public.git" 22 | ) 23 | 24 | func TestNewGitHubParserWithURL(t *testing.T) { 25 | { 26 | gl, err := NewBitBucketParserWithURL(urlA) 27 | assert.NoError(t, err) 28 | assert.Equal(t, "bitbucket.org", gl.GetHostName()) 29 | assert.Equal(t, "bitbucket", gl.GetProvider()) 30 | assert.Equal(t, "matthyx", gl.GetOwnerName()) 31 | assert.Equal(t, "ks-testing-public", gl.GetRepoName()) 32 | assert.Equal(t, urlA, gl.GetURL().String()) 33 | assert.Equal(t, "", gl.GetBranchName()) 34 | assert.Equal(t, "", gl.GetPath()) 35 | } 36 | { 37 | gl, err := NewBitBucketParserWithURL(urlB) 38 | assert.NoError(t, err) 39 | assert.Equal(t, "bitbucket.org", gl.GetHostName()) 40 | assert.Equal(t, "bitbucket", gl.GetProvider()) 41 | assert.Equal(t, "matthyx", gl.GetOwnerName()) 42 | assert.Equal(t, "ks-testing-public", gl.GetRepoName()) 43 | assert.Equal(t, urlA, gl.GetURL().String()) 44 | assert.Equal(t, "master", gl.GetBranchName()) 45 | assert.Equal(t, "rules/etcd-encryption-native/raw.rego", gl.GetPath()) 46 | } 47 | { 48 | gl, err := NewBitBucketParserWithURL(urlC) 49 | assert.NoError(t, err) 50 | assert.Equal(t, "bitbucket.org", gl.GetHostName()) 51 | assert.Equal(t, "bitbucket", gl.GetProvider()) 52 | assert.Equal(t, "matthyx", gl.GetOwnerName()) 53 | assert.Equal(t, "ks-testing-public", gl.GetRepoName()) 54 | assert.Equal(t, urlA, gl.GetURL().String()) 55 | assert.Equal(t, "dev", gl.GetBranchName()) 56 | assert.Equal(t, "README.md", gl.GetPath()) 57 | } 58 | { 59 | gl, err := NewBitBucketParserWithURL(urlD) 60 | assert.NoError(t, err) 61 | assert.Equal(t, "bitbucket.org", gl.GetHostName()) 62 | assert.Equal(t, "bitbucket", gl.GetProvider()) 63 | assert.Equal(t, "matthyx", gl.GetOwnerName()) 64 | assert.Equal(t, "ks-testing-public", gl.GetRepoName()) 65 | assert.Equal(t, urlA, gl.GetURL().String()) 66 | assert.Equal(t, "dev", gl.GetBranchName()) 67 | assert.Equal(t, "", gl.GetPath()) 68 | } 69 | { 70 | gl, err := NewBitBucketParserWithURL(urlE) 71 | assert.NoError(t, err) 72 | assert.Equal(t, "bitbucket.org", gl.GetHostName()) 73 | assert.Equal(t, "bitbucket", gl.GetProvider()) 74 | assert.Equal(t, "matthyx", gl.GetOwnerName()) 75 | assert.Equal(t, "ks-testing-public", gl.GetRepoName()) 76 | assert.Equal(t, urlA, gl.GetURL().String()) 77 | assert.Equal(t, "v1.0.178", gl.GetBranchName()) 78 | assert.Equal(t, "README.md", gl.GetPath()) 79 | } 80 | { 81 | gl, err := NewBitBucketParserWithURL(urlF) 82 | assert.NoError(t, err) 83 | assert.Equal(t, "bitbucket.org", gl.GetHostName()) 84 | assert.Equal(t, "bitbucket", gl.GetProvider()) 85 | assert.Equal(t, "matthyx", gl.GetOwnerName()) 86 | assert.Equal(t, "ks-testing-public", gl.GetRepoName()) 87 | assert.Equal(t, urlA, gl.GetURL().String()) 88 | assert.Equal(t, "4502b9b51ee3ac1ea649bacfa0f48ebdeab05f4a", gl.GetBranchName()) 89 | assert.Equal(t, "README.md", gl.GetPath()) 90 | } 91 | { 92 | gl, err := NewBitBucketParserWithURL(urlG) 93 | assert.NoError(t, err) 94 | assert.Equal(t, "bitbucket.org", gl.GetHostName()) 95 | assert.Equal(t, "bitbucket", gl.GetProvider()) 96 | assert.Equal(t, "matthyx", gl.GetOwnerName()) 97 | assert.Equal(t, "ks-testing-public", gl.GetRepoName()) 98 | assert.Equal(t, "https://bitbucket.org/matthyx/ks-testing-public", gl.GetURL().String()) 99 | assert.Equal(t, "", gl.GetBranchName()) 100 | assert.Equal(t, "", gl.GetPath()) 101 | } 102 | { 103 | gl, err := NewBitBucketParserWithURL(urlH) 104 | assert.NoError(t, err) 105 | assert.Equal(t, "bitbucket.org", gl.GetHostName()) 106 | assert.Equal(t, "bitbucket", gl.GetProvider()) 107 | assert.Equal(t, "matthyx", gl.GetOwnerName()) 108 | assert.Equal(t, "to", gl.GetRepoName()) 109 | assert.Equal(t, "https://bitbucket.org/matthyx/to", gl.GetURL().String()) 110 | assert.Equal(t, "ks-testing-public.git", gl.GetBranchName()) // invalid input leads to incorrect guess. At least this does not panic. 111 | assert.Equal(t, "", gl.GetPath()) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /apis/azureapi/mockapis.go: -------------------------------------------------------------------------------- 1 | package azureapi 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | ) 7 | 8 | var mockTreeUrl = `{ "count": 12, "value": [ { "objectId": "3838adf7667bcec12a8bd0b868e5ce18e4476815", "gitObjectType": "tree", "commitId": "8cb368a261285f7d5129109f9bb3918de1999449", "path": "/", "isFolder": true, "url": "https://dev.azure.com/anubhav06/2773a1e0-3a09-4b3c-962d-31b884fe58c4/_apis/git/repositories/e4a1a3d3-c2e7-4781-89af-2dff3a8412f6/items?path=%2F&versionType=Branch&version=dev&versionOptions=None" }, { "objectId": "62c893550adb53d3a8fc29a1584ff831cb829062", "gitObjectType": "blob", "commitId": "8cb368a261285f7d5129109f9bb3918de1999449", "path": "/.gitignore", "url": "https://dev.azure.com/anubhav06/testing/_apis/git/repositories/e4a1a3d3-c2e7-4781-89af-2dff3a8412f6/items//.gitignore?versionType=Branch&version=dev&versionOptions=None" }, { "objectId": "9172c84f0c2141833cc3b0ed512f94e36e789aa0", "gitObjectType": "blob", "commitId": "8cb368a261285f7d5129109f9bb3918de1999449", "path": "/README.md", "url": "https://dev.azure.com/anubhav06/testing/_apis/git/repositories/e4a1a3d3-c2e7-4781-89af-2dff3a8412f6/items//README.md?versionType=Branch&version=dev&versionOptions=None" }, { "objectId": "ea6048a012f2a69a09660cea3138ac64db49c021", "gitObjectType": "blob", "commitId": "8cb368a261285f7d5129109f9bb3918de1999449", "path": "/postgres-pod.yml", "url": "https://dev.azure.com/anubhav06/testing/_apis/git/repositories/e4a1a3d3-c2e7-4781-89af-2dff3a8412f6/items//postgres-pod.yml?versionType=Branch&version=dev&versionOptions=None" }, { "objectId": "2f16bbce9eee40d071aed4bd9a0e601fc99b9216", "gitObjectType": "blob", "commitId": "8cb368a261285f7d5129109f9bb3918de1999449", "path": "/postgres-service.yml", "url": "https://dev.azure.com/anubhav06/testing/_apis/git/repositories/e4a1a3d3-c2e7-4781-89af-2dff3a8412f6/items//postgres-service.yml?versionType=Branch&version=dev&versionOptions=None" }, { "objectId": "a0af8b4a8fe692a4526a159e851a7bf7efde1c69", "gitObjectType": "blob", "commitId": "8cb368a261285f7d5129109f9bb3918de1999449", "path": "/redis-pod.yml", "url": "https://dev.azure.com/anubhav06/testing/_apis/git/repositories/e4a1a3d3-c2e7-4781-89af-2dff3a8412f6/items//redis-pod.yml?versionType=Branch&version=dev&versionOptions=None" }, { "objectId": "2bab8621dfb18f40b5487df40aaef3921e61849b", "gitObjectType": "blob", "commitId": "8cb368a261285f7d5129109f9bb3918de1999449", "path": "/redis-service.yml", "url": "https://dev.azure.com/anubhav06/testing/_apis/git/repositories/e4a1a3d3-c2e7-4781-89af-2dff3a8412f6/items//redis-service.yml?versionType=Branch&version=dev&versionOptions=None" }, { "objectId": "009faa3b128a0375772f4914b74419da3abd079b", "gitObjectType": "blob", "commitId": "8cb368a261285f7d5129109f9bb3918de1999449", "path": "/result-app-pod.yml", "url": "https://dev.azure.com/anubhav06/testing/_apis/git/repositories/e4a1a3d3-c2e7-4781-89af-2dff3a8412f6/items//result-app-pod.yml?versionType=Branch&version=dev&versionOptions=None" }, { "objectId": "523ad3a6ac0419d335ffa3269bc4e057d06fe63e", "gitObjectType": "blob", "commitId": "8cb368a261285f7d5129109f9bb3918de1999449", "path": "/result-app-service.yml", "url": "https://dev.azure.com/anubhav06/testing/_apis/git/repositories/e4a1a3d3-c2e7-4781-89af-2dff3a8412f6/items//result-app-service.yml?versionType=Branch&version=dev&versionOptions=None" }, { "objectId": "bcced3583f5c8a208ecf7149a195ad21d17083aa", "gitObjectType": "blob", "commitId": "8cb368a261285f7d5129109f9bb3918de1999449", "path": "/voting-app-pod.yml", "url": "https://dev.azure.com/anubhav06/testing/_apis/git/repositories/e4a1a3d3-c2e7-4781-89af-2dff3a8412f6/items//voting-app-pod.yml?versionType=Branch&version=dev&versionOptions=None" }, { "objectId": "d055a91baadb058e7ed211705f5e4b8f42127215", "gitObjectType": "blob", "commitId": "8cb368a261285f7d5129109f9bb3918de1999449", "path": "/voting-app-service.yml", "url": "https://dev.azure.com/anubhav06/testing/_apis/git/repositories/e4a1a3d3-c2e7-4781-89af-2dff3a8412f6/items//voting-app-service.yml?versionType=Branch&version=dev&versionOptions=None" }, { "objectId": "ea912ca1e7a8bf4bd4b149d68f97f692c3a11092", "gitObjectType": "blob", "commitId": "8cb368a261285f7d5129109f9bb3918de1999449", "path": "/worker-app-pod.yml", "url": "https://dev.azure.com/anubhav06/testing/_apis/git/repositories/e4a1a3d3-c2e7-4781-89af-2dff3a8412f6/items//worker-app-pod.yml?versionType=Branch&version=dev&versionOptions=None" } ] }` 9 | var mockLastCommit = `{ "count": 1, "value": [ { "commitId": "8cb368a261285f7d5129109f9bb3918de1999449", "author": { "name": "Anubhav Gupta", "email": null, "date": "2022-12-22T12:58:10Z" }, "committer": { "name": "Anubhav Gupta", "email": null, "date": "2022-12-22T12:58:10Z" }, "comment": "Updated README.md", "changeCounts": { "Add": 0, "Edit": 1, "Delete": 0 }, "url": "https://dev.azure.com/anubhav06/2773a1e0-3a09-4b3c-962d-31b884fe58c4/_apis/git/repositories/e4a1a3d3-c2e7-4781-89af-2dff3a8412f6/commits/8cb368a261285f7d5129109f9bb3918de1999449", "remoteUrl": "https://dev.azure.com/anubhav06/testing/_git/testing/commit/8cb368a261285f7d5129109f9bb3918de1999449" } ] }` 10 | 11 | type MockAzureAPI struct { 12 | } 13 | 14 | func NewMockAzureAPI() *MockAzureAPI { return &MockAzureAPI{} } 15 | 16 | func (az *MockAzureAPI) GetRepoTree(owner, project, repo, branch string, headers *Headers) (*Tree, error) { 17 | t := Tree{} 18 | switch fmt.Sprintf("%s/%s/_git/%s", owner, project, repo) { 19 | case "anubhav06/testing/_git/testing": 20 | json.Unmarshal([]byte(mockTreeUrl), &t) 21 | } 22 | return &t, nil 23 | } 24 | 25 | func (az MockAzureAPI) GetDefaultBranchName(owner, project, repo string, headers *Headers) (string, error) { 26 | return "master", nil 27 | } 28 | 29 | func (az MockAzureAPI) GetLatestCommit(owner, project, repo, branch string, headers *Headers) (*Commit, error) { 30 | 31 | var data Commit 32 | 33 | if err := json.Unmarshal([]byte(mockLastCommit), &data); err != nil { 34 | return &data, err 35 | } 36 | return &data, nil 37 | } 38 | -------------------------------------------------------------------------------- /apis/azureapi/apis.go: -------------------------------------------------------------------------------- 1 | package azureapi 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | 9 | "github.com/kubescape/go-git-url/apis" 10 | ) 11 | 12 | const ( 13 | DEFAULT_HOST string = "azure.com" 14 | DEV_HOST string = "dev.azure.com" 15 | SSH_DEV_HOST string = "ssh.dev.azure.com" 16 | ) 17 | 18 | type IAzureAPI interface { 19 | GetRepoTree(owner, project, repo, branch string, headers *Headers) (*Tree, error) 20 | GetDefaultBranchName(owner, project, repo string, headers *Headers) (string, error) 21 | GetLatestCommit(owner, project, repo, branch string, headers *Headers) (*Commit, error) 22 | } 23 | 24 | type AzureAPI struct { 25 | httpClient *http.Client 26 | } 27 | 28 | func NewAzureAPI() *AzureAPI { return &AzureAPI{httpClient: &http.Client{}} } 29 | 30 | func (az *AzureAPI) GetRepoTree(owner, project, repo, branch string, headers *Headers) (*Tree, error) { 31 | treeAPI := APIRepoTree(owner, project, repo, branch) 32 | body, err := apis.HttpGet(az.httpClient, treeAPI, headers.ToMap()) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | var tree Tree 38 | if err = json.Unmarshal([]byte(body), &tree); err != nil { 39 | return nil, fmt.Errorf("failed to unmarshal response body from '%s', reason: %s", treeAPI, err.Error()) 40 | } 41 | 42 | return &tree, nil 43 | 44 | } 45 | 46 | func (az *AzureAPI) GetDefaultBranchName(owner, project, repo string, headers *Headers) (string, error) { 47 | 48 | body, err := apis.HttpGet(az.httpClient, APIMetadata(owner, project, repo), headers.ToMap()) 49 | if err != nil { 50 | return "", err 51 | } 52 | 53 | var data azureDefaultBranchAPI 54 | err = json.Unmarshal([]byte(body), &data) 55 | if err != nil { 56 | return "", err 57 | } 58 | for i := range data.BranchValue { 59 | if data.BranchValue[i].IsBaseVersion { 60 | return data.BranchValue[i].Name, nil 61 | } 62 | } 63 | 64 | errAPI := errors.New("unable to find default branch from the Azure API") 65 | return "", errAPI 66 | } 67 | 68 | func (az *AzureAPI) GetLatestCommit(owner, project, repo, branch string, headers *Headers) (*Commit, error) { 69 | 70 | body, err := apis.HttpGet(az.httpClient, APILastCommitsOfBranch(owner, project, repo, branch), headers.ToMap()) 71 | if err != nil { 72 | return nil, err 73 | } 74 | 75 | var data Commit 76 | err = json.Unmarshal([]byte(body), &data) 77 | if err != nil { 78 | return &data, err 79 | } 80 | return &data, nil 81 | } 82 | 83 | // Get latest commit data of path/file 84 | func (az *AzureAPI) GetFileLatestCommit(owner, project, repo, branch, fullPath string, headers *Headers) ([]Commit, error) { 85 | 86 | body, err := apis.HttpGet(az.httpClient, APILastCommitsOfPath(owner, project, repo, branch, fullPath), headers.ToMap()) 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | var data []Commit 92 | err = json.Unmarshal([]byte(body), &data) 93 | if err != nil { 94 | return data, err 95 | } 96 | return data, nil 97 | } 98 | 99 | // APIRepoTree Azure tree api 100 | // API Ref: https://learn.microsoft.com/en-us/rest/api/azure/devops/git/items/list?view=azure-devops-rest-7.0&tabs=HTTP#full-recursion-and-with-content-metadata 101 | // Example: https://dev.azure.com/anubhav06/testing/_apis/git/repositories/testing/items?recursionLevel=Full&versionDescriptor.version=dev&api-version=5.1 102 | func APIRepoTree(owner, project, repo, branch string) string { 103 | return fmt.Sprintf("https://dev.%s/%s/%s/_apis/git/repositories/%s/items?recursionLevel=Full&versionDescriptor.version=%s&api-version=5.1", DEFAULT_HOST, owner, project, repo, branch) 104 | } 105 | 106 | // APIRaw Azure raw file api 107 | // API Ref: https://learn.microsoft.com/en-us/rest/api/azure/devops/build/source-providers/get-file-contents?view=azure-devops-rest-7.0 108 | // https://stackoverflow.com/questions/56281152/how-to-get-a-link-to-a-file-from-a-vso-repo/56283730#56283730 109 | // Example: https://dev.azure.com/anubhav06/k8s-example/_apis/sourceProviders/tfsgit/filecontents?&repository=k8s-example&commitOrBranch=master&path=/volumes/cephfs/cephfs.yaml&api-version=7.0 110 | func APIRaw(owner, project, repo, branch, path string) string { 111 | return fmt.Sprintf("https://dev.%s/%s/%s/_apis/sourceProviders/tfsgit/filecontents?&repository=%s&commitOrBranch=%s&path=%s", DEFAULT_HOST, owner, project, repo, branch, path) 112 | } 113 | 114 | // APIDefaultBranch Azure repo metadata api 115 | // API Ref: https://learn.microsoft.com/en-us/rest/api/azure/devops/git/stats/list?view=azure-devops-rest-4.1&tabs=HTTP 116 | // Example: https://dev.azure.com/anubhav06/k8s-example/_apis/git/repositories/k8s-example/stats/branches?api-version=4.1 117 | func APIMetadata(owner, project, repo string) string { 118 | return fmt.Sprintf("https://dev.%s/%s/%s/_apis/git/repositories/%s/stats/branches?api-version=4.1", DEFAULT_HOST, owner, project, repo) 119 | } 120 | 121 | // APILastCommits Azure last commit api 122 | // API Ref: https://learn.microsoft.com/en-us/rest/api/azure/devops/git/commits/get-commits?view=azure-devops-rest-4.1&tabs=HTTP 123 | // Example: https://dev.azure.com/anubhav06/k8s-example/_apis/git/repositories/k8s-example/commits?searchCriteria.$top=1&api-version=4.1 124 | func APILastCommits(owner, project, repo string) string { 125 | return fmt.Sprintf("https://dev.%s/%s/%s/_apis/git/repositories/%s/commits?searchCriteria.$top=1", DEFAULT_HOST, owner, project, repo) 126 | } 127 | 128 | // APILastCommitsOfBranch Azure last commit of specific branch api 129 | // API Ref: https://learn.microsoft.com/en-us/rest/api/azure/devops/git/commits/get-commits?view=azure-devops-rest-4.1&tabs=HTTP 130 | // Example: https://dev.azure.com/anubhav06/testing/_apis/git/repositories/testing/commits?searchCriteria.$top=1&searchCriteria.itemVersion.version=dev 131 | func APILastCommitsOfBranch(owner, project, repo, branch string) string { 132 | return fmt.Sprintf("%s&searchCriteria.itemVersion.version=%s", APILastCommits(owner, project, repo), branch) 133 | } 134 | 135 | // APILastCommitsOfPath Azure last commit of specific branch api 136 | // API Ref: https://learn.microsoft.com/en-us/rest/api/azure/devops/git/commits/get-commits?view=azure-devops-rest-4.1&tabs=HTTP 137 | // Example: https://dev.azure.com/anubhav06/k8s-example/_apis/git/repositories/k8s-example/commits?searchCriteria.$top=1&searchCriteria.itemVersion.version=master&searchCriteria.itemPath=volumes/storageos/storageos-pod.yaml 138 | func APILastCommitsOfPath(owner, project, repo, branch, path string) string { 139 | return fmt.Sprintf("%s&searchCriteria.itemPath=%s", APILastCommitsOfBranch(owner, project, repo, branch), path) 140 | } 141 | -------------------------------------------------------------------------------- /init_test.go: -------------------------------------------------------------------------------- 1 | package giturl 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestNewGitURL(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | fullURL string 14 | provider string 15 | owner string 16 | repo string 17 | branch string 18 | url string 19 | wantErr assert.ErrorAssertionFunc 20 | }{ 21 | { 22 | name: "parse github", 23 | fullURL: "https://github.com/kubescape/go-git-url", 24 | provider: "github", 25 | owner: "kubescape", 26 | repo: "go-git-url", 27 | url: "https://github.com/kubescape/go-git-url", 28 | wantErr: assert.NoError, 29 | }, 30 | { 31 | name: "parse github www", 32 | fullURL: "https://www.github.com/kubescape/go-git-url", 33 | provider: "github", 34 | owner: "kubescape", 35 | repo: "go-git-url", 36 | url: "https://github.com/kubescape/go-git-url", 37 | wantErr: assert.NoError, 38 | }, 39 | { 40 | name: "parse github ssh", 41 | fullURL: "git@github.com:kubescape/go-git-url.git", 42 | provider: "github", 43 | owner: "kubescape", 44 | repo: "go-git-url", 45 | url: "https://github.com/kubescape/go-git-url", 46 | wantErr: assert.NoError, 47 | }, 48 | { 49 | name: "parse azure", 50 | fullURL: "https://dev.azure.com/dwertent/ks-testing-public/_git/ks-testing-public", 51 | provider: "azure", 52 | owner: "dwertent", 53 | repo: "ks-testing-public", 54 | url: "https://dev.azure.com/dwertent/ks-testing-public/_git/ks-testing-public", 55 | wantErr: assert.NoError, 56 | }, 57 | { 58 | name: "parse azure ssh", 59 | fullURL: "git@ssh.dev.azure.com:v3/dwertent/ks-testing-public/ks-testing-public", 60 | provider: "azure", 61 | owner: "dwertent", 62 | repo: "ks-testing-public", 63 | url: "https://dev.azure.com/dwertent/ks-testing-public/_git/ks-testing-public", 64 | wantErr: assert.NoError, 65 | }, 66 | { 67 | name: "parse bitbucket https", 68 | fullURL: "https://bitbucket.org/matthyx/ks-testing-public.git", 69 | provider: "bitbucket", 70 | owner: "matthyx", 71 | repo: "ks-testing-public", 72 | url: "https://bitbucket.org/matthyx/ks-testing-public", 73 | wantErr: assert.NoError, 74 | }, 75 | { 76 | name: "parse bitbucket ssh", 77 | fullURL: "git@bitbucket.org:matthyx/ks-testing-public.git", 78 | provider: "bitbucket", 79 | owner: "matthyx", 80 | repo: "ks-testing-public", 81 | url: "https://bitbucket.org/matthyx/ks-testing-public", 82 | wantErr: assert.NoError, 83 | }, 84 | { 85 | name: "parse gitlab", 86 | fullURL: "https://gitlab.com/kubescape/testing", 87 | provider: "gitlab", 88 | owner: "kubescape", 89 | repo: "testing", 90 | url: "https://gitlab.com/kubescape/testing", 91 | wantErr: assert.NoError, 92 | }, 93 | { 94 | name: "parse gitlab ssh", 95 | fullURL: "git@gitlab.com:kubescape/testing.git", 96 | provider: "gitlab", 97 | owner: "kubescape", 98 | repo: "testing", 99 | url: "https://gitlab.com/kubescape/testing", 100 | wantErr: assert.NoError, 101 | }, 102 | { 103 | name: "parse gitlab branch", 104 | fullURL: "https://gitlab.com/kubescape/testing/-/tree/dev", 105 | provider: "gitlab", 106 | owner: "kubescape", 107 | repo: "testing", 108 | branch: "dev", 109 | url: "https://gitlab.com/kubescape/testing", 110 | wantErr: assert.NoError, 111 | }, 112 | { 113 | name: "repo only", 114 | fullURL: "https://git.host.com/repo.git", 115 | provider: "gitlab", 116 | owner: "", 117 | repo: "repo", 118 | url: "https://git.host.com/repo", 119 | wantErr: assert.NoError, 120 | }, 121 | } 122 | for _, tt := range tests { 123 | t.Run(tt.name, func(t *testing.T) { 124 | gh, err := NewGitURL(tt.fullURL) 125 | if !tt.wantErr(t, err, fmt.Sprintf("NewGitURL(%v)", tt.fullURL)) { 126 | return 127 | } 128 | assert.Equal(t, tt.provider, gh.GetProvider()) 129 | assert.Equal(t, tt.owner, gh.GetOwnerName()) 130 | assert.Equal(t, tt.repo, gh.GetRepoName()) 131 | assert.Equal(t, tt.branch, gh.GetBranchName()) 132 | assert.Equal(t, tt.url, gh.GetURL().String()) 133 | }) 134 | } 135 | } 136 | 137 | func TestNewGitAPI(t *testing.T) { 138 | fileText := "https://raw.githubusercontent.com/kubescape/go-git-url/master/files/file0.text" 139 | var gitURL IGitAPI 140 | var err error 141 | { 142 | gitURL, err = NewGitAPI("https://github.com/kubescape/go-git-url") 143 | assert.NoError(t, err) 144 | 145 | files, err := gitURL.ListFilesNamesWithExtension([]string{"yaml", "json"}) 146 | assert.NoError(t, err) 147 | assert.Equal(t, 8, len(files)) 148 | } 149 | 150 | { 151 | gitURL, err = NewGitAPI("https://github.com/kubescape/go-git-url") 152 | assert.NoError(t, err) 153 | 154 | files, errM := gitURL.DownloadFilesWithExtension([]string{"text"}) 155 | assert.Equal(t, 0, len(errM)) 156 | assert.Equal(t, 1, len(files)) 157 | assert.Equal(t, "name=file0", string(files[fileText])) 158 | 159 | } 160 | 161 | { 162 | gitURL, err = NewGitAPI(fileText) 163 | assert.NoError(t, err) 164 | 165 | files, errM := gitURL.DownloadFilesWithExtension([]string{"text"}) 166 | assert.Equal(t, 0, len(errM)) 167 | assert.Equal(t, 1, len(files)) 168 | assert.Equal(t, "name=file0", string(files[fileText])) 169 | } 170 | 171 | { 172 | gitURL, err = NewGitAPI(fileText) 173 | assert.NoError(t, err) 174 | 175 | files, errM := gitURL.DownloadAllFiles() 176 | assert.Equal(t, 0, len(errM)) 177 | assert.Equal(t, 1, len(files)) 178 | assert.Equal(t, "name=file0", string(files[fileText])) 179 | } 180 | 181 | { 182 | gitURL, err := NewGitAPI("https://github.com/kubescape/go-git-url/tree/master/files") 183 | assert.NoError(t, err) 184 | 185 | files, errM := gitURL.DownloadFilesWithExtension([]string{"text"}) 186 | assert.Equal(t, 0, len(errM)) 187 | assert.Equal(t, 1, len(files)) 188 | assert.Equal(t, "name=file0", string(files[fileText])) 189 | 190 | } 191 | 192 | { 193 | gitURL, err = NewGitAPI("https://github.com/kubescape/go-git-url/blob/master/files/file0.text") 194 | assert.NoError(t, err) 195 | 196 | files, errM := gitURL.DownloadFilesWithExtension([]string{"text"}) 197 | assert.Equal(t, 0, len(errM)) 198 | assert.Equal(t, 1, len(files)) 199 | assert.Equal(t, "name=file0", string(files[fileText])) 200 | 201 | } 202 | 203 | { 204 | gitURL, err = NewGitAPI("https://gitlab.host.com/kubescape/testing") 205 | assert.NoError(t, err) 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /apis/githubapi/mockapis.go: -------------------------------------------------------------------------------- 1 | package githubapi 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | ) 7 | 8 | var mockTreeUrl = `{"sha":"e03c6e27ef88e1bd30902b7a6258f0a54e01c2c7","url":"https://api.github.com/repos/kubescape/go-git-url/git/trees/e03c6e27ef88e1bd30902b7a6258f0a54e01c2c7","tree":[{"path":".vscode","mode":"040000","type":"tree","sha":"d58bd8e47da3a8fb24ffd678916563e1cdf1ebaa","url":"https://api.github.com/repos/kubescape/go-git-url/git/trees/d58bd8e47da3a8fb24ffd678916563e1cdf1ebaa"},{"path":".vscode/settings.json","mode":"100644","type":"blob","sha":"a7e16169143195e2a711f051bdbfcfb08088db2e","size":54,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/a7e16169143195e2a711f051bdbfcfb08088db2e"},{"path":"LICENSE","mode":"100644","type":"blob","sha":"261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64","size":11357,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64"},{"path":"README.md","mode":"100644","type":"blob","sha":"ea1a34a54f88f528e6864bc7cbf32cf2d794b964","size":1033,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/ea1a34a54f88f528e6864bc7cbf32cf2d794b964"},{"path":"apis","mode":"040000","type":"tree","sha":"702eeb95f9ed5b124ce84a96a9205f49b7c7379c","url":"https://api.github.com/repos/kubescape/go-git-url/git/trees/702eeb95f9ed5b124ce84a96a9205f49b7c7379c"},{"path":"apis/githubapi","mode":"040000","type":"tree","sha":"eebffec78929c432e2c7f63d9c53e5ef3de9d54c","url":"https://api.github.com/repos/kubescape/go-git-url/git/trees/eebffec78929c432e2c7f63d9c53e5ef3de9d54c"},{"path":"apis/githubapi/apis.go","mode":"100644","type":"blob","sha":"ba744632d38f23a3e8813ea1a9ab55c58f2ca450","size":1360,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/ba744632d38f23a3e8813ea1a9ab55c58f2ca450"},{"path":"apis/githubapi/datastructures.go","mode":"100644","type":"blob","sha":"57286350dd1cc21f9074c566c685046005299d35","size":256,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/57286350dd1cc21f9074c566c685046005299d35"},{"path":"apis/githubapi/methods.go","mode":"100644","type":"blob","sha":"c71f7fe3f62757473aef30a7613fbb2052bbe7ef","size":426,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/c71f7fe3f62757473aef30a7613fbb2052bbe7ef"},{"path":"apis/http.go","mode":"100644","type":"blob","sha":"e65dcd117797fd02fa7b02a7d4813a15cc6e7289","size":1869,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/e65dcd117797fd02fa7b02a7d4813a15cc6e7289"},{"path":"files","mode":"040000","type":"tree","sha":"57c59318121a8918f6ff259cdd3306c249215fdc","url":"https://api.github.com/repos/kubescape/go-git-url/git/trees/57c59318121a8918f6ff259cdd3306c249215fdc"},{"path":"files/file0.json","mode":"100644","type":"blob","sha":"ae85c07ec156f563e2d4184ce292219321f415ec","size":17,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/ae85c07ec156f563e2d4184ce292219321f415ec"},{"path":"files/file0.text","mode":"100644","type":"blob","sha":"e25bb064ec2d827c3a4a3a93797642e6d9b9b1f3","size":10,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/e25bb064ec2d827c3a4a3a93797642e6d9b9b1f3"},{"path":"files/file0.yaml","mode":"100644","type":"blob","sha":"31358e0ec5b5136961e0795962cdd1a0b51307ce","size":11,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/31358e0ec5b5136961e0795962cdd1a0b51307ce"},{"path":"files/file1.json","mode":"100644","type":"blob","sha":"41a0cb5ad5cadd8d12df5c8f493a41f370cab03d","size":17,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/41a0cb5ad5cadd8d12df5c8f493a41f370cab03d"},{"path":"files/file1.yaml","mode":"100644","type":"blob","sha":"cda23e65facdc68d2ea60a94715053085bacf97f","size":11,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/cda23e65facdc68d2ea60a94715053085bacf97f"},{"path":"files/file2.yaml","mode":"100644","type":"blob","sha":"177289fb9bc6aa7b19336111e902f92c8b4d28e7","size":11,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/177289fb9bc6aa7b19336111e902f92c8b4d28e7"},{"path":"githubparser","mode":"040000","type":"tree","sha":"7ca01c978d838116cca40c2fbe2a3743219abd43","url":"https://api.github.com/repos/kubescape/go-git-url/git/trees/7ca01c978d838116cca40c2fbe2a3743219abd43"},{"path":"githubparser/v1","mode":"040000","type":"tree","sha":"f7498c41927bb478faa94294ced8ad0eb0022448","url":"https://api.github.com/repos/kubescape/go-git-url/git/trees/f7498c41927bb478faa94294ced8ad0eb0022448"},{"path":"githubparser/v1/datastructures.go","mode":"100644","type":"blob","sha":"3567877260d4cdad12eeb9facac72e5e4597348b","size":250,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/3567877260d4cdad12eeb9facac72e5e4597348b"},{"path":"githubparser/v1/parser.go","mode":"100644","type":"blob","sha":"bafc7b5cba10cf094ebdd054158b3cfdaa4389cc","size":3997,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/bafc7b5cba10cf094ebdd054158b3cfdaa4389cc"},{"path":"go.mod","mode":"100644","type":"blob","sha":"f5832d5bdf68408a5a8e21f51f6f3d1bf5b8e956","size":103,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/f5832d5bdf68408a5a8e21f51f6f3d1bf5b8e956"},{"path":"go.sum","mode":"100644","type":"blob","sha":"207ee15dca443269452c6c96d4b4df62c48e7e5e","size":974,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/207ee15dca443269452c6c96d4b4df62c48e7e5e"},{"path":"init.go","mode":"100644","type":"blob","sha":"6b0a179812d9df99602f2563bcd759af1920b809","size":638,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/6b0a179812d9df99602f2563bcd759af1920b809"},{"path":"interface.go","mode":"100644","type":"blob","sha":"9ccf2806ef1f897d0ebf5278b3353ca25d7d1288","size":401,"url":"https://api.github.com/repos/kubescape/go-git-url/git/blobs/9ccf2806ef1f897d0ebf5278b3353ca25d7d1288"}],"truncated":false}` 9 | var mockLastCommit = `{"sha":"e7d287e491b4002bc59d67ad7423d8119fc89e6c","node_id":"C_kwDOHK4XZ9oAKGU3ZDI4N2U0OTFiNDAwMmJjNTlkNjdhZDc0MjNkODExOWZjODllNmM","commit":{"author":{"name":"David Wertenteil","email":"dwertent@armosec.io","date":"2022-04-13T11:51:06Z"},"committer":{"name":"David Wertenteil","email":"dwertent@armosec.io","date":"2022-04-13T11:51:06Z"},"message":"updated interface","tree":{"sha":"fa28de192d078a748266ca0faf2ec0663ec40748","url":"https://api.github.com/repos/kubescape/go-git-url/git/trees/fa28de192d078a748266ca0faf2ec0663ec40748"},"url":"https://api.github.com/repos/kubescape/go-git-url/git/commits/e7d287e491b4002bc59d67ad7423d8119fc89e6c","comment_count":0,"verification":{"verified":false,"reason":"unsigned","signature":null,"payload":null}},"url":"https://api.github.com/repos/kubescape/go-git-url/commits/e7d287e491b4002bc59d67ad7423d8119fc89e6c","html_url":"https://github.com/kubescape/go-git-url/commit/e7d287e491b4002bc59d67ad7423d8119fc89e6c","comments_url":"https://api.github.com/repos/kubescape/go-git-url/commits/e7d287e491b4002bc59d67ad7423d8119fc89e6c/comments","author":{"login":"dwertent","id":64066841,"node_id":"MDQ6VXNlcjY0MDY2ODQx","avatar_url":"https://avatars.githubusercontent.com/u/64066841?v=4","gravatar_id":"","url":"https://api.github.com/users/dwertent","html_url":"https://github.com/dwertent","followers_url":"https://api.github.com/users/dwertent/followers","following_url":"https://api.github.com/users/dwertent/following{/other_user}","gists_url":"https://api.github.com/users/dwertent/gists{/gist_id}","starred_url":"https://api.github.com/users/dwertent/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwertent/subscriptions","organizations_url":"https://api.github.com/users/dwertent/orgs","repos_url":"https://api.github.com/users/dwertent/repos","events_url":"https://api.github.com/users/dwertent/events{/privacy}","received_events_url":"https://api.github.com/users/dwertent/received_events","type":"User","site_admin":false},"committer":{"login":"dwertent","id":64066841,"node_id":"MDQ6VXNlcjY0MDY2ODQx","avatar_url":"https://avatars.githubusercontent.com/u/64066841?v=4","gravatar_id":"","url":"https://api.github.com/users/dwertent","html_url":"https://github.com/dwertent","followers_url":"https://api.github.com/users/dwertent/followers","following_url":"https://api.github.com/users/dwertent/following{/other_user}","gists_url":"https://api.github.com/users/dwertent/gists{/gist_id}","starred_url":"https://api.github.com/users/dwertent/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwertent/subscriptions","organizations_url":"https://api.github.com/users/dwertent/orgs","repos_url":"https://api.github.com/users/dwertent/repos","events_url":"https://api.github.com/users/dwertent/events{/privacy}","received_events_url":"https://api.github.com/users/dwertent/received_events","type":"User","site_admin":false},"parents":[{"sha":"f7ffc4f4dc6b2793f12e4dd708a2fe1ac95fbe8f","url":"https://api.github.com/repos/kubescape/go-git-url/commits/f7ffc4f4dc6b2793f12e4dd708a2fe1ac95fbe8f","html_url":"https://github.com/kubescape/go-git-url/commit/f7ffc4f4dc6b2793f12e4dd708a2fe1ac95fbe8f"}],"stats":{"total":24,"additions":15,"deletions":9},"files":[{"sha":"f0c7b60d418d66a1a724afd5471b7eeb9bcb5900","filename":"init_test.go","status":"modified","additions":7,"deletions":5,"changes":12,"blob_url":"https://github.com/kubescape/go-git-url/blob/e7d287e491b4002bc59d67ad7423d8119fc89e6c/init_test.go","raw_url":"https://github.com/kubescape/go-git-url/raw/e7d287e491b4002bc59d67ad7423d8119fc89e6c/init_test.go","contents_url":"https://api.github.com/repos/kubescape/go-git-url/contents/init_test.go?ref=e7d287e491b4002bc59d67ad7423d8119fc89e6c","patch":"@@ -8,8 +8,10 @@ import (\n \n func TestNewGitURL(t *testing.T) {\n \tfileText := \"https://raw.githubusercontent.com/kubescape/go-git-url/master/files/file0.text\"\n+\tvar gitURL IGitURL\n+\tvar err error\n \t{\n-\t\tgitURL, err := NewGitURL(\"https://github.com/kubescape/go-git-url\")\n+\t\tgitURL, err = NewGitURL(\"https://github.com/kubescape/go-git-url\")\n \t\tassert.NoError(t, err)\n \n \t\tfiles, err := gitURL.ListFilesNamesWithExtension([]string{\"yaml\", \"json\"})\n@@ -18,7 +20,7 @@ func TestNewGitURL(t *testing.T) {\n \t}\n \n \t{\n-\t\tgitURL, err := NewGitURL(\"https://github.com/kubescape/go-git-url\")\n+\t\tgitURL, err = NewGitURL(\"https://github.com/kubescape/go-git-url\")\n \t\tassert.NoError(t, err)\n \n \t\tfiles, errM := gitURL.DownloadFilesWithExtension([]string{\"text\"})\n@@ -29,7 +31,7 @@ func TestNewGitURL(t *testing.T) {\n \t}\n \n \t{\n-\t\tgitURL, err := NewGitURL(fileText)\n+\t\tgitURL, err = NewGitURL(fileText)\n \t\tassert.NoError(t, err)\n \n \t\tfiles, errM := gitURL.DownloadFilesWithExtension([]string{\"text\"})\n@@ -39,7 +41,7 @@ func TestNewGitURL(t *testing.T) {\n \t}\n \n \t{\n-\t\tgitURL, err := NewGitURL(fileText)\n+\t\tgitURL, err = NewGitURL(fileText)\n \t\tassert.NoError(t, err)\n \n \t\tfiles, errM := gitURL.DownloadAllFiles()\n@@ -60,7 +62,7 @@ func TestNewGitURL(t *testing.T) {\n \t}\n \n \t{\n-\t\tgitURL, err := NewGitURL(\"https://github.com/kubescape/go-git-url/blob/master/files/file0.text\")\n+\t\tgitURL, err = NewGitURL(\"https://github.com/kubescape/go-git-url/blob/master/files/file0.text\")\n \t\tassert.NoError(t, err)\n \n \t\tfiles, errM := gitURL.DownloadFilesWithExtension([]string{\"text\"})"},{"sha":"e3a8ec5fee6cf2c0450eb5466380a9afed8212d2","filename":"interface.go","status":"modified","additions":8,"deletions":4,"changes":12,"blob_url":"https://github.com/kubescape/go-git-url/blob/e7d287e491b4002bc59d67ad7423d8119fc89e6c/interface.go","raw_url":"https://github.com/kubescape/go-git-url/raw/e7d287e491b4002bc59d67ad7423d8119fc89e6c/interface.go","contents_url":"https://api.github.com/repos/kubescape/go-git-url/contents/interface.go?ref=e7d287e491b4002bc59d67ad7423d8119fc89e6c","patch":"@@ -4,19 +4,23 @@ import \"net/url\"\n \n // IGitURL parse git urls\n type IGitURL interface {\n-\n-\t// parse url\n-\tParse(fullURL string) error\n-\n \tSetBranch(string)\n \tSetOwner(string)\n \tSetPath(string)\n \tSetToken(string)\n+\tSetRepo(string)\n \n \tGetBranch() string\n \tGetOwner() string\n \tGetPath() string\n \tGetToken() string\n+\tGetRepo() string\n+\n+\t// parse url\n+\tParse(fullURL string) error\n+\n+\t// set default branch from the API\n+\tSetDefaultBranch() error\n \n \t// GetURL git url\n \tGetURL() *url.URL"}]}` 10 | 11 | type MockGitHubAPI struct { 12 | } 13 | 14 | func NewMockGitHubAPI() *MockGitHubAPI { return &MockGitHubAPI{} } 15 | 16 | func (gh *MockGitHubAPI) GetRepoTree(owner, repo, branch string, headers *Headers) (*Tree, error) { 17 | t := &Tree{} 18 | switch fmt.Sprintf("%s/%s", owner, repo) { 19 | case "kubescape/go-git-url": 20 | json.Unmarshal([]byte(mockTreeUrl), t) 21 | } 22 | return t, nil 23 | } 24 | 25 | func (gh MockGitHubAPI) GetDefaultBranchName(owner, repo string, headers *Headers) (string, error) { 26 | return "master", nil 27 | } 28 | 29 | func (gh MockGitHubAPI) GetLatestCommit(owner, repo, branch string, headers *Headers) (*Commit, error) { 30 | 31 | var data Commit 32 | 33 | if err := json.Unmarshal([]byte(mockLastCommit), &data); err != nil { 34 | return &data, err 35 | } 36 | return &data, nil 37 | } 38 | --------------------------------------------------------------------------------