├── .gitignore ├── testutil ├── test-fixtures │ ├── example │ │ ├── tree │ │ │ └── bang │ │ ├── foo │ │ └── types │ │ │ └── bar.json │ └── example.json └── git_repo.go ├── config ├── test-fixtures │ ├── invalid_config.json │ └── local.json ├── load_test.go ├── mock │ └── mock.go ├── config.go └── load.go ├── Godeps ├── Readme └── Godeps.json ├── version.go ├── CHANGELOG.md ├── repository ├── ref.go ├── clone_test.go ├── mock │ └── mock.go ├── clone.go ├── repository_test.go ├── pull_test.go ├── load.go ├── checkout_test.go ├── ref_test.go ├── diff_test.go ├── diff.go ├── checkout.go ├── pull.go ├── load_test.go └── repository.go ├── glide.yaml ├── LICENSE.md ├── glide.lock ├── kv ├── ref.go ├── kv.go ├── handler.go ├── update.go ├── branch.go └── init.go ├── Makefile ├── watcher ├── interval_test.go ├── interval.go ├── watcher.go └── webhook.go ├── TODO.md ├── main.go ├── runner └── runner.go └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor 2 | /build/bin 3 | -------------------------------------------------------------------------------- /testutil/test-fixtures/example/tree/bang: -------------------------------------------------------------------------------- 1 | This is bang's content 2 | -------------------------------------------------------------------------------- /testutil/test-fixtures/example/foo: -------------------------------------------------------------------------------- 1 | This is foo's content update 69 2 | -------------------------------------------------------------------------------- /testutil/test-fixtures/example/types/bar.json: -------------------------------------------------------------------------------- 1 | { 2 | "key_1": "value_1" 3 | } 4 | -------------------------------------------------------------------------------- /config/test-fixtures/invalid_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "repos": [ 3 | { 4 | "name": "example" 5 | } 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /Godeps/Readme: -------------------------------------------------------------------------------- 1 | This directory tree is generated automatically by godep. 2 | 3 | Please do not edit. 4 | 5 | See https://github.com/tools/godep for more information. 6 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // The git commit that will be used to describe the version 4 | var ( 5 | GitCommit string 6 | ) 7 | 8 | // Version of the program 9 | const Version = "0.0.1" 10 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.1.0 (UNRELEASED) 2 | 3 | * Initial version of go-git2consul that contains basic functionality 4 | * Handle repository files to KV, no support for file extensions yet 5 | * Handle tracking branches 6 | * Interval and webhook polling 7 | * Handle CRUD operations, and perform updates on deltas 8 | -------------------------------------------------------------------------------- /testutil/test-fixtures/example.json: -------------------------------------------------------------------------------- 1 | { 2 | "repos": [ 3 | { 4 | "name": "test-example", 5 | "url": "./test-fixtures/example", 6 | "branches": [ 7 | "master" 8 | ], 9 | "hooks": [ 10 | { 11 | "type": "polling", 12 | "interval": 5 13 | } 14 | ] 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /config/test-fixtures/local.json: -------------------------------------------------------------------------------- 1 | { 2 | "repos": [ 3 | { 4 | "name": "test-example", 5 | "url": "./test-fixtures/example", 6 | "branches": [ 7 | "master", 8 | "test" 9 | ], 10 | "hooks": [ 11 | { 12 | "type": "polling", 13 | "interval": 5 14 | } 15 | ] 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /repository/ref.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import "gopkg.in/libgit2/git2go.v24" 4 | 5 | // CheckRef checks whether a particular ref is part of the repository 6 | func (r *Repository) CheckRef(ref string) error { 7 | oid, err := git.NewOid(ref) 8 | if err != nil { 9 | return err 10 | } 11 | 12 | // This can be for a different repo 13 | _, err = r.Lookup(oid) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | return nil 19 | } 20 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/Cimpress-MCP/go-git2consul 2 | import: 3 | - package: github.com/hashicorp/consul 4 | subpackages: 5 | - api 6 | - package: github.com/hashicorp/go-cleanhttp 7 | - package: github.com/hashicorp/serf 8 | subpackages: 9 | - coordinate 10 | - package: golang.org/x/sys 11 | subpackages: 12 | - unix 13 | - package: gopkg.in/libgit2/git2go.v24 14 | - package: github.com/apex/log 15 | subpackages: 16 | - handlers/text 17 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2016 Cimpress 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 4 | 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | -------------------------------------------------------------------------------- /config/load_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/apex/log" 8 | "github.com/apex/log/handlers/discard" 9 | ) 10 | 11 | func init() { 12 | log.SetHandler(discard.New()) 13 | } 14 | 15 | func TestLoad(t *testing.T) { 16 | file := filepath.Join("test-fixtures", "local.json") 17 | 18 | _, err := Load(file) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | } 23 | 24 | func TestLoad_invalidConfig(t *testing.T) { 25 | file := filepath.Join("test-fixtures", "invalid_config.json") 26 | 27 | _, err := Load(file) 28 | if err == nil { 29 | t.Fatal("Expected failure") 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /repository/clone_test.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "os" 5 | "path" 6 | "testing" 7 | 8 | "github.com/Cimpress-MCP/go-git2consul/config/mock" 9 | "github.com/Cimpress-MCP/go-git2consul/testutil" 10 | ) 11 | 12 | func TestClone(t *testing.T) { 13 | gitRepo, cleanup := testutil.GitInitTestRepo(t) 14 | defer cleanup() 15 | 16 | cfg := mock.Config(gitRepo.Workdir()) 17 | 18 | repo := &Repository{ 19 | Config: cfg.Repos[0], 20 | } 21 | 22 | repoPath := path.Join(cfg.LocalStore, repo.Config.Name) 23 | err := repo.Clone(repoPath) 24 | if err != nil { 25 | t.Fatal(err) 26 | } 27 | 28 | //Cleanup cloned repo 29 | defer func() { 30 | err = os.RemoveAll(repo.Workdir()) 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | }() 35 | } 36 | -------------------------------------------------------------------------------- /repository/mock/mock.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | 7 | "github.com/Cimpress-MCP/go-git2consul/config/mock" 8 | "github.com/Cimpress-MCP/go-git2consul/repository" 9 | 10 | "gopkg.in/libgit2/git2go.v24" 11 | ) 12 | 13 | // Repository returns a mock of a repository.Repository object 14 | func Repository(gitRepo *git.Repository) *repository.Repository { 15 | if gitRepo == nil { 16 | return nil 17 | } 18 | 19 | repoConfig := mock.RepoConfig(gitRepo.Workdir()) 20 | 21 | dstPath, err := ioutil.TempDir("", "git2consul-test-local") 22 | if err != nil { 23 | fmt.Println(err) 24 | return nil 25 | } 26 | 27 | localRepo, err := git.Clone(repoConfig.Url, dstPath, &git.CloneOptions{}) 28 | if err != nil { 29 | fmt.Print(err) 30 | return nil 31 | } 32 | 33 | repo := &repository.Repository{ 34 | Repository: localRepo, 35 | Config: repoConfig, 36 | } 37 | 38 | return repo 39 | } 40 | -------------------------------------------------------------------------------- /repository/clone.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "fmt" 5 | 6 | "gopkg.in/libgit2/git2go.v24" 7 | ) 8 | 9 | // Clone the repository. Cloning will only checkout tracked branches. 10 | // A destination path to clone to needs to be provided 11 | func (r *Repository) Clone(path string) error { 12 | r.Lock() 13 | defer r.Unlock() 14 | 15 | // Clone the first tracked branch instead of the default branch 16 | if len(r.Config.Branches) == 0 { 17 | return fmt.Errorf("No tracked branches specified") 18 | } 19 | checkoutBranch := r.Config.Branches[0] 20 | 21 | rawRepo, err := git.Clone(r.Config.Url, path, &git.CloneOptions{ 22 | CheckoutOpts: &git.CheckoutOpts{ 23 | Strategy: git.CheckoutNone, 24 | }, 25 | CheckoutBranch: checkoutBranch, 26 | }) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | r.Repository = rawRepo 32 | 33 | err = r.checkoutConfigBranches() 34 | if err != nil { 35 | return err 36 | } 37 | 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /repository/repository_test.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/Cimpress-MCP/go-git2consul/config/mock" 8 | "github.com/Cimpress-MCP/go-git2consul/testutil" 9 | ) 10 | 11 | func TestNew(t *testing.T) { 12 | gitRepo, cleanup := testutil.GitInitTestRepo(t) 13 | defer cleanup() 14 | 15 | cfg := mock.Config(gitRepo.Workdir()) 16 | repoConfig := cfg.Repos[0] 17 | 18 | repo, status, err := New(cfg.LocalStore, repoConfig) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | if status != RepositoryCloned { 24 | t.Fatalf("Expected clone status") 25 | } 26 | 27 | // Call New() again, this time expecting RepositoryOpened 28 | repo, status, err = New(cfg.LocalStore, repoConfig) 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | 33 | if status != RepositoryOpened { 34 | t.Fatalf("Expected clone status") 35 | } 36 | 37 | // Cleanup cloning 38 | defer func() { 39 | err := os.RemoveAll(repo.Workdir()) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | }() 44 | } 45 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: fe700a632becb3b3104deb4c8cdb12c30fed7554bf7d2efc43eb240d93be43a2 2 | updated: 2016-05-24T17:32:28.314043612-04:00 3 | imports: 4 | - name: github.com/apex/log 5 | version: a999c1b29c986b1972ec53b15092b63a8051ec83 6 | subpackages: 7 | - handlers/text 8 | - name: github.com/gorilla/context 9 | version: a8d44e7d8e4d532b6a27a02dd82abb31cc1b01bd 10 | - name: github.com/gorilla/mux 11 | version: 9c19ed558d5df4da88e2ade9c8940d742aef0e7e 12 | - name: github.com/hashicorp/consul 13 | version: 14c24154e8db989a8d17fd5e15f1b6ce7885f29e 14 | subpackages: 15 | - api 16 | - name: github.com/hashicorp/go-cleanhttp 17 | version: 875fb671b3ddc66f8e2f0acc33829c8cb989a38d 18 | - name: github.com/hashicorp/serf 19 | version: e4ec8cc423bbe20d26584b96efbeb9102e16d05f 20 | subpackages: 21 | - coordinate 22 | - name: golang.org/x/sys 23 | version: d4feaf1a7e61e1d9e79e6c4e76c6349e9cab0a03 24 | subpackages: 25 | - unix 26 | - name: gopkg.in/libgit2/git2go.v24 27 | version: 8eaae73f85dd3df78df80d2dac066eb0866444ae 28 | devImports: [] 29 | -------------------------------------------------------------------------------- /repository/pull_test.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/Cimpress-MCP/go-git2consul/config/mock" 9 | "github.com/Cimpress-MCP/go-git2consul/testutil" 10 | "gopkg.in/libgit2/git2go.v24" 11 | ) 12 | 13 | func TestPull(t *testing.T) { 14 | gitRepo, cleanup := testutil.GitInitTestRepo(t) 15 | defer cleanup() 16 | 17 | repoConfig := mock.RepoConfig(gitRepo.Workdir()) 18 | dstPath := filepath.Join(os.TempDir(), repoConfig.Name) 19 | 20 | localRepo, err := git.Clone(repoConfig.Url, dstPath, &git.CloneOptions{}) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | 25 | repo := &Repository{ 26 | Repository: localRepo, 27 | Config: repoConfig, 28 | } 29 | 30 | // Push a commit to the repository 31 | testutil.GitCommitTestRepo(t) 32 | 33 | _, err = repo.Pull("master") 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | 38 | // Cleanup 39 | defer func() { 40 | err = os.RemoveAll(repo.Workdir()) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | }() 45 | } 46 | -------------------------------------------------------------------------------- /Godeps/Godeps.json: -------------------------------------------------------------------------------- 1 | { 2 | "ImportPath": "github.com/Cimpress-MCP/go-git2consul", 3 | "GoVersion": "go1.6", 4 | "GodepVersion": "v62", 5 | "Deps": [ 6 | { 7 | "ImportPath": "github.com/Sirupsen/logrus", 8 | "Comment": "v0.9.0-17-ga26f435", 9 | "Rev": "a26f43589d737684363ff856c5a0f9f24b946510" 10 | }, 11 | { 12 | "ImportPath": "github.com/hashicorp/consul/api", 13 | "Comment": "v0.6.4-241-g7637116", 14 | "Rev": "763711686ae954b603d58a0980180ab5a910655d" 15 | }, 16 | { 17 | "ImportPath": "github.com/hashicorp/go-cleanhttp", 18 | "Rev": "ad28ea4487f05916463e2423a55166280e8254b5" 19 | }, 20 | { 21 | "ImportPath": "github.com/hashicorp/serf/coordinate", 22 | "Comment": "v0.7.0-58-gdefb069", 23 | "Rev": "defb069b1bad9f7cdebc647810cb6ae398a1b617" 24 | }, 25 | { 26 | "ImportPath": "golang.org/x/sys/unix", 27 | "Rev": "f64b50fbea64174967a8882830d621a18ee1548e" 28 | }, 29 | { 30 | "ImportPath": "gopkg.in/libgit2/git2go.v24", 31 | "Rev": "8eaae73f85dd3df78df80d2dac066eb0866444ae" 32 | } 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /repository/load.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/Cimpress-MCP/go-git2consul/config" 7 | "github.com/apex/log" 8 | ) 9 | 10 | // LoadRepos populates Repository slice from configuration. It also 11 | // handles cloning of the repository if not present 12 | func LoadRepos(cfg *config.Config) ([]*Repository, error) { 13 | logger := log.WithFields(log.Fields{ 14 | "caller": "repository", 15 | }) 16 | repos := []*Repository{} 17 | 18 | // Create Repository object for each repo 19 | for _, repoConfig := range cfg.Repos { 20 | r, state, err := New(cfg.LocalStore, repoConfig) 21 | if err != nil { 22 | return nil, fmt.Errorf("Error loading %s: %s", repoConfig.Name, err) 23 | } 24 | 25 | switch state { 26 | case RepositoryCloned: 27 | logger.Infof("Cloned repository %s", r.Name()) 28 | case RepositoryOpened: 29 | logger.Infof("Loaded repository %s", r.Name()) 30 | } 31 | 32 | repos = append(repos, r) 33 | } 34 | 35 | if len(repos) == 0 { 36 | return repos, fmt.Errorf("No repositories provided in the configuration") 37 | } 38 | 39 | return repos, nil 40 | } 41 | -------------------------------------------------------------------------------- /repository/checkout_test.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/Cimpress-MCP/go-git2consul/config/mock" 9 | "github.com/Cimpress-MCP/go-git2consul/testutil" 10 | "gopkg.in/libgit2/git2go.v24" 11 | ) 12 | 13 | func TestCheckoutBranch(t *testing.T) { 14 | gitRepo, cleanup := testutil.GitInitTestRepo(t) 15 | defer cleanup() 16 | 17 | repoConfig := mock.RepoConfig(gitRepo.Workdir()) 18 | dstPath := filepath.Join(os.TempDir(), repoConfig.Name) 19 | 20 | localRepo, err := git.Clone(repoConfig.Url, dstPath, &git.CloneOptions{}) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | 25 | repo := &Repository{ 26 | Repository: localRepo, 27 | Config: repoConfig, 28 | } 29 | 30 | branch, err := repo.LookupBranch("master", git.BranchLocal) 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | 35 | err = repo.CheckoutBranch(branch, &git.CheckoutOpts{}) 36 | if err != nil { 37 | t.Fatal(err) 38 | } 39 | 40 | // Cleanup 41 | defer func() { 42 | err = os.RemoveAll(repo.Workdir()) 43 | if err != nil { 44 | t.Fatal(err) 45 | } 46 | }() 47 | } 48 | -------------------------------------------------------------------------------- /config/mock/mock.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "os" 5 | "time" 6 | 7 | "github.com/Cimpress-MCP/go-git2consul/config" 8 | ) 9 | 10 | // RepoConfig returns a mock Repo config object 11 | func RepoConfig(repoUrl string) *config.Repo { 12 | return &config.Repo{ 13 | Name: "git2consul-test-local", 14 | Url: repoUrl, 15 | Branches: []string{"master"}, 16 | Hooks: []*config.Hook{ 17 | { 18 | Type: "polling", 19 | Interval: 5 * time.Second, 20 | }, 21 | }, 22 | } 23 | } 24 | 25 | // Config returns a mock Config object with one repository configuration 26 | func Config(repoUrl string) *config.Config { 27 | return &config.Config{ 28 | LocalStore: os.TempDir(), 29 | HookSvr: &config.HookSvrConfig{ 30 | Port: 9000, 31 | }, 32 | Repos: []*config.Repo{ 33 | { 34 | Name: "git2consul-test-local", 35 | Url: repoUrl, 36 | Branches: []string{"master"}, 37 | Hooks: []*config.Hook{ 38 | { 39 | Type: "polling", 40 | Interval: 5 * time.Second, 41 | }, 42 | }, 43 | }, 44 | }, 45 | Consul: &config.ConsulConfig{ 46 | Address: "127.0.0.1:8500", 47 | }, 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /kv/ref.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "fmt" 5 | "path" 6 | 7 | "github.com/Cimpress-MCP/go-git2consul/repository" 8 | "github.com/hashicorp/consul/api" 9 | ) 10 | 11 | // Get local branch ref from the KV 12 | func (h *KVHandler) getKVRef(repo *repository.Repository, branchName string) (string, error) { 13 | refFile := fmt.Sprintf("%s.ref", branchName) 14 | key := path.Join(repo.Name(), refFile) 15 | 16 | pair, _, err := h.Get(key, nil) 17 | if err != nil { 18 | return "", err 19 | } 20 | 21 | // If error on get, return empty value 22 | if pair == nil { 23 | return "", nil 24 | } 25 | 26 | return string(pair.Value), nil 27 | } 28 | 29 | // Put the local branch ref to the KV 30 | func (h *KVHandler) putKVRef(repo *repository.Repository, branchName string) error { 31 | refFile := fmt.Sprintf("%s.ref", branchName) 32 | key := path.Join(repo.Name(), refFile) 33 | 34 | rawRef, err := repo.References.Lookup("refs/heads/" + branchName) 35 | if err != nil { 36 | return err 37 | } 38 | ref := rawRef.Target().String() 39 | 40 | p := &api.KVPair{ 41 | Key: key, 42 | Value: []byte(ref), 43 | } 44 | 45 | _, err = h.Put(p, nil) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /repository/ref_test.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/Cimpress-MCP/go-git2consul/config/mock" 9 | "github.com/Cimpress-MCP/go-git2consul/testutil" 10 | "gopkg.in/libgit2/git2go.v24" 11 | ) 12 | 13 | func TestCheckRef(t *testing.T) { 14 | gitRepo, cleanup := testutil.GitInitTestRepo(t) 15 | defer cleanup() 16 | 17 | repoConfig := mock.RepoConfig(gitRepo.Workdir()) 18 | dstPath := filepath.Join(os.TempDir(), repoConfig.Name) 19 | 20 | localRepo, err := git.Clone(repoConfig.Url, dstPath, &git.CloneOptions{}) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | 25 | repo := &Repository{ 26 | Repository: localRepo, 27 | Config: repoConfig, 28 | } 29 | 30 | h, err := repo.Head() 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | 35 | ref := h.Target().String() 36 | 37 | // Push a commit to the repository 38 | testutil.GitCommitTestRepo(t) 39 | 40 | _, err = repo.Pull("master") 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | err = repo.CheckRef(ref) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | 50 | // Cleanup 51 | defer func() { 52 | err = os.RemoveAll(repo.Workdir()) 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | }() 57 | } 58 | -------------------------------------------------------------------------------- /kv/kv.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "io/ioutil" 5 | "path" 6 | "path/filepath" 7 | 8 | "github.com/Cimpress-MCP/go-git2consul/repository" 9 | "github.com/hashicorp/consul/api" 10 | ) 11 | 12 | func (h *KVHandler) putKV(repo *repository.Repository, prefix string) error { 13 | head, err := repo.Head() 14 | if err != nil { 15 | return err 16 | } 17 | 18 | branchName, err := head.Branch().Name() 19 | if err != nil { 20 | return err 21 | } 22 | 23 | key := path.Join(repo.Name(), branchName, prefix) 24 | filePath := filepath.Join(repo.Workdir(), prefix) 25 | value, err := ioutil.ReadFile(filePath) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | p := &api.KVPair{ 31 | Key: key, 32 | Value: value, 33 | } 34 | 35 | _, err = h.Put(p, nil) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | return nil 41 | } 42 | 43 | func (h *KVHandler) deleteKV(repo *repository.Repository, prefix string) error { 44 | head, err := repo.Head() 45 | if err != nil { 46 | return err 47 | } 48 | 49 | branchName, err := head.Branch().Name() 50 | if err != nil { 51 | return err 52 | } 53 | 54 | key := path.Join(repo.Name(), branchName, prefix) 55 | 56 | _, err = h.Delete(key, nil) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | return nil 62 | } 63 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TEST?=$(shell go list ./... | grep -v /vendor/) 2 | 3 | # Get git commit information 4 | GIT_COMMIT=$(shell git rev-parse HEAD) 5 | GIT_DIRTY=$(shell test -n "`git status --porcelain`" && echo "+CHANGES" || true) 6 | 7 | default: test 8 | 9 | test: generate 10 | @echo " ==> Running tests..." 11 | @go list $(TEST) \ 12 | | grep -v "/vendor/" \ 13 | | xargs -n1 go test -v -timeout=60s $(TESTARGS) 14 | .PHONY: test 15 | 16 | generate: 17 | @echo " ==> Generating..." 18 | @find . -type f -name '.DS_Store' -delete 19 | @go list ./... \ 20 | | grep -v "/vendor/" \ 21 | | xargs -n1 go generate $(PACKAGES) 22 | .PHONY: generate 23 | 24 | 25 | build: generate 26 | @echo " ==> Building..." 27 | @go build -ldflags "-X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY}" . 28 | .PHONY: build 29 | 30 | build-linux: create-build-image remove-dangling build-native 31 | .PHONY: build-linux 32 | 33 | create-build-image: 34 | @docker build -t cimpress/git2consul-builder $(CURDIR)/build/ 35 | .PHONY: create-build-image 36 | 37 | remove-dangling: 38 | @docker images --quiet --filter dangling=true | grep . | xargs docker rmi 39 | .PHONY: remove-dangling 40 | 41 | run-build-image: 42 | @echo " ===> Building..." 43 | @docker run --rm --name git2consul-builder -v $(CURDIR):/app -v $(CURDIR)/build/bin:/build/bin --entrypoint /app/build/build.sh cimpress/git2consul-builder 44 | .PHONY: run-build-image 45 | -------------------------------------------------------------------------------- /watcher/interval_test.go: -------------------------------------------------------------------------------- 1 | package watch 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/Cimpress-MCP/go-git2consul/repository" 8 | "github.com/Cimpress-MCP/go-git2consul/repository/mock" 9 | "github.com/Cimpress-MCP/go-git2consul/testutil" 10 | "github.com/apex/log" 11 | "github.com/apex/log/handlers/discard" 12 | ) 13 | 14 | func init() { 15 | log.SetHandler(discard.New()) 16 | } 17 | 18 | func TestPollBranches(t *testing.T) { 19 | gitRepo, cleanup := testutil.GitInitTestRepo(t) 20 | defer cleanup() 21 | 22 | repo := mock.Repository(gitRepo) 23 | 24 | oid, _ := testutil.GitCommitTestRepo(t) 25 | odb, err := repo.Odb() 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | 30 | w := &Watcher{ 31 | Repositories: []*repository.Repository{repo}, 32 | RepoChangeCh: make(chan *repository.Repository, 1), 33 | ErrCh: make(chan error), 34 | RcvDoneCh: make(chan struct{}, 1), 35 | SndDoneCh: make(chan struct{}, 1), 36 | logger: log.WithField("caller", "watcher"), 37 | hookSvr: nil, 38 | once: true, 39 | } 40 | 41 | err = w.pollBranches(repo) 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | 46 | if !odb.Exists(oid) { 47 | t.Fatal("Commit not present on remote") 48 | } 49 | 50 | // Cleanup on git2consul cached repo 51 | defer func() { 52 | err = os.RemoveAll(repo.Workdir()) 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | }() 57 | } 58 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "time" 4 | 5 | // Hook is the configuration for hooks 6 | type Hook struct { 7 | Type string `json:"type"` 8 | 9 | // Specific to polling 10 | Interval time.Duration `json:"interval"` 11 | 12 | // Specific to webhooks 13 | Url string `json:"url,omitempty"` 14 | } 15 | 16 | // Repo is the configuration for the repository 17 | type Repo struct { 18 | Name string `json:"name"` 19 | Url string `json:"url"` 20 | Branches []string `json:"branches"` 21 | Hooks []*Hook `json:"hooks"` 22 | } 23 | 24 | // Config is used to represent the passed in configuration 25 | type Config struct { 26 | LocalStore string `json:"local_store"` 27 | HookSvr *HookSvrConfig `json:"webhook"` 28 | Repos []*Repo `json:"repos"` 29 | Consul *ConsulConfig `json:"consul"` 30 | } 31 | 32 | // HookSvrConfig is the configuration for the git hoooks server 33 | type HookSvrConfig struct { 34 | Address string `json:"address,omitempty"` 35 | Port int `json:"port"` 36 | } 37 | 38 | // ConsulConfig is the configuration for the Consul client 39 | type ConsulConfig struct { 40 | Address string `json:"address"` 41 | Token string `json:"token,omitempty"` 42 | SSLEnable bool `json:"ssl"` 43 | SSLVerify bool `json:"ssl_verify,omitempty"` 44 | } 45 | 46 | func (r *Repo) String() string { 47 | if r != nil { 48 | return r.Name 49 | } 50 | return "" 51 | } 52 | -------------------------------------------------------------------------------- /kv/handler.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "crypto/tls" 5 | "net/http" 6 | 7 | "github.com/Cimpress-MCP/go-git2consul/config" 8 | "github.com/apex/log" 9 | "github.com/hashicorp/consul/api" 10 | ) 11 | 12 | // KVHandler is used to manipulate the KV 13 | type KVHandler struct { 14 | *api.KV 15 | logger *log.Entry 16 | } 17 | 18 | // New creates new KV handler to manipulate the Consul VK 19 | func New(config *config.ConsulConfig) (*KVHandler, error) { 20 | client, err := newAPIClient(config) 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | logger := log.WithFields(log.Fields{ 26 | "caller": "consul", 27 | }) 28 | 29 | kv := client.KV() 30 | 31 | handler := &KVHandler{ 32 | KV: kv, 33 | logger: logger, 34 | } 35 | 36 | return handler, nil 37 | } 38 | 39 | func newAPIClient(config *config.ConsulConfig) (*api.Client, error) { 40 | consulConfig := api.DefaultConfig() 41 | 42 | if config.Address != "" { 43 | consulConfig.Address = config.Address 44 | } 45 | 46 | if config.Token != "" { 47 | consulConfig.Token = config.Token 48 | } 49 | 50 | if config.SSLEnable { 51 | consulConfig.Scheme = "https" 52 | } 53 | 54 | if !config.SSLVerify { 55 | consulConfig.HttpClient.Transport = &http.Transport{ 56 | TLSClientConfig: &tls.Config{ 57 | InsecureSkipVerify: true, 58 | }, 59 | } 60 | } 61 | 62 | client, err := api.NewClient(consulConfig) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | return client, nil 68 | } 69 | -------------------------------------------------------------------------------- /repository/diff_test.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/Cimpress-MCP/go-git2consul/config/mock" 9 | "github.com/Cimpress-MCP/go-git2consul/testutil" 10 | "gopkg.in/libgit2/git2go.v24" 11 | ) 12 | 13 | func TestDiffStatus(t *testing.T) { 14 | gitRepo, cleanup := testutil.GitInitTestRepo(t) 15 | defer cleanup() 16 | 17 | repoConfig := mock.RepoConfig(gitRepo.Workdir()) 18 | dstPath := filepath.Join(os.TempDir(), repoConfig.Name) 19 | 20 | localRepo, err := git.Clone(repoConfig.Url, dstPath, &git.CloneOptions{}) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | 25 | repo := &Repository{ 26 | Repository: localRepo, 27 | Config: repoConfig, 28 | } 29 | 30 | h, err := repo.Head() 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | 35 | oldRef := h.Target().String() 36 | 37 | // Push a commit to the repository 38 | testutil.GitCommitTestRepo(t) 39 | 40 | _, err = repo.Pull("master") 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | deltas, err := repo.DiffStatus(oldRef) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | 50 | if len(deltas) == 0 { 51 | t.Fatal("Expected deltas from pull changes") 52 | } 53 | 54 | if deltas[0].Status != git.DeltaModified { 55 | t.Fatalf("Expected DeltaModified on %s", deltas[0].OldFile.Path) 56 | } 57 | 58 | // Cleanup 59 | defer func() { 60 | err = os.RemoveAll(repo.Workdir()) 61 | if err != nil { 62 | t.Fatal(err) 63 | } 64 | }() 65 | } 66 | -------------------------------------------------------------------------------- /repository/diff.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import "gopkg.in/libgit2/git2go.v24" 4 | 5 | // DiffStatus compares the current workdir with a target ref and return the modified files 6 | func (r *Repository) DiffStatus(ref string) ([]git.DiffDelta, error) { 7 | deltas := []git.DiffDelta{} 8 | 9 | oid, err := git.NewOid(ref) 10 | if err != nil { 11 | return nil, err 12 | } 13 | 14 | // This can be for a different repo 15 | obj, err := r.Lookup(oid) 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | commit, err := obj.AsCommit() 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | tree, err := commit.Tree() 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | h, err := r.Head() 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | obj2, err := r.Lookup(h.Target()) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | commit2, err := obj2.AsCommit() 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | tree2, err := commit2.Tree() 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | do, err := git.DefaultDiffOptions() 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | diffs, err := r.DiffTreeToTree(tree, tree2, &do) 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | n, err := diffs.NumDeltas() 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | for i := 0; i < n; i++ { 66 | diff, err := diffs.GetDelta(i) 67 | if err != nil { 68 | return nil, err 69 | } 70 | deltas = append(deltas, diff) 71 | } 72 | 73 | return deltas, nil 74 | } 75 | -------------------------------------------------------------------------------- /kv/update.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "github.com/Cimpress-MCP/go-git2consul/repository" 5 | "github.com/apex/log" 6 | ) 7 | 8 | // HandleUpdate handles the update of a particular repository by 9 | // comparing diffs against the KV. 10 | func (h *KVHandler) HandleUpdate(repo *repository.Repository) error { 11 | repo.Lock() 12 | defer repo.Unlock() 13 | 14 | head, err := repo.Head() 15 | if err != nil { 16 | return err 17 | } 18 | b, err := head.Branch().Name() 19 | if err != nil { 20 | return err 21 | } 22 | 23 | h.logger.Infof("KV GET ref: %s/%s", repo.Name(), b) 24 | kvRef, err := h.getKVRef(repo, b) 25 | if err != nil { 26 | return err 27 | } 28 | 29 | // Local ref 30 | localRef := head.Target().String() 31 | // log.Debugf("(consul) kvRef: %s | localRef: %s", kvRef, localRef) 32 | 33 | if len(kvRef) == 0 { 34 | log.Infof("KV PUT changes: %s/%s", repo.Name(), b) 35 | err := h.putBranch(repo, head.Branch()) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | err = h.putKVRef(repo, b) 41 | if err != nil { 42 | return err 43 | } 44 | h.logger.Infof("KV PUT ref: %s/%s", repo.Name(), b) 45 | } else if kvRef != localRef { 46 | // Check if the ref belongs to that repo 47 | err := repo.CheckRef(kvRef) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | // Handle modified and deleted files 53 | deltas, err := repo.DiffStatus(kvRef) 54 | if err != nil { 55 | return err 56 | } 57 | h.handleDeltas(repo, deltas) 58 | 59 | err = h.putKVRef(repo, b) 60 | if err != nil { 61 | return err 62 | } 63 | h.logger.Infof("KV PUT ref: %s/%s", repo.Name(), b) 64 | } 65 | 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /repository/checkout.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "path" 5 | 6 | "gopkg.in/libgit2/git2go.v24" 7 | ) 8 | 9 | func stringInSlice(a string, list []string) bool { 10 | for _, b := range list { 11 | if b == a { 12 | return true 13 | } 14 | } 15 | return false 16 | } 17 | 18 | // Checkout branches specified in the config 19 | func (r *Repository) checkoutConfigBranches() error { 20 | itr, err := r.NewBranchIterator(git.BranchRemote) 21 | if err != nil { 22 | return err 23 | } 24 | defer itr.Free() 25 | 26 | var checkoutBranchFn = func(b *git.Branch, _ git.BranchType) error { 27 | bn, err := b.Name() 28 | if err != nil { 29 | return err 30 | } 31 | 32 | // Only checkout tracked branches 33 | // TODO: optimize this O(n^2) 34 | if stringInSlice(path.Base(bn), r.Config.Branches) == false { 35 | return nil 36 | } 37 | 38 | _, err = r.References.Lookup("refs/heads/" + path.Base(bn)) 39 | if err != nil { 40 | localRef, err := r.References.Create("refs/heads/"+path.Base(bn), b.Reference.Target(), true, "") 41 | if err != nil { 42 | return err 43 | } 44 | 45 | err = r.SetHead(localRef.Name()) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | err = r.CheckoutHead(&git.CheckoutOpts{ 51 | Strategy: git.CheckoutForce, 52 | }) 53 | if err != nil { 54 | return err 55 | } 56 | } 57 | 58 | return nil 59 | } 60 | 61 | err = itr.ForEach(checkoutBranchFn) 62 | 63 | if err != nil && !git.IsErrorCode(err, git.ErrIterOver) { 64 | return err 65 | } 66 | 67 | return nil 68 | } 69 | 70 | // CheckoutBranch performs a checkout on a specific branch 71 | func (r *Repository) CheckoutBranch(branch *git.Branch, opts *git.CheckoutOpts) error { 72 | err := r.SetHead(branch.Reference.Name()) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | err = r.CheckoutHead(opts) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | return nil 83 | } 84 | -------------------------------------------------------------------------------- /kv/branch.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path" 7 | "path/filepath" 8 | "strings" 9 | 10 | "github.com/Cimpress-MCP/go-git2consul/repository" 11 | "github.com/apex/log" 12 | "github.com/hashicorp/consul/api" 13 | "gopkg.in/libgit2/git2go.v24" 14 | ) 15 | 16 | // Push a repository branch to the KV 17 | // TODO: Optimize for PUT only on changes instead of the entire repo 18 | func (h *KVHandler) putBranch(repo *repository.Repository, branch *git.Branch) error { 19 | // Checkout branch 20 | repo.CheckoutBranch(branch, &git.CheckoutOpts{ 21 | Strategy: git.CheckoutForce, 22 | }) 23 | 24 | // h, _ := repo.Head() 25 | // bn, _ := h.Branch().Name() 26 | // log.Debugf("(consul) pushBranch(): Branch: %s Head: %s", bn, h.Target().String()) 27 | 28 | var pushFile = func(fullpath string, info os.FileInfo, err error) error { 29 | // Walk error 30 | if err != nil { 31 | return err 32 | } 33 | 34 | // Skip the .git directory 35 | if info.IsDir() && info.Name() == ".git" { 36 | return filepath.SkipDir 37 | } 38 | 39 | // Do not push directories 40 | if info.IsDir() { 41 | return nil 42 | } 43 | 44 | // KV path, is repo/branch/file 45 | branchName, err := branch.Name() 46 | if err != nil { 47 | return err 48 | } 49 | 50 | key := strings.TrimPrefix(fullpath, repo.Workdir()) 51 | kvPath := path.Join(repo.Name(), branchName, key) 52 | h.logger.Debugf("KV PUT changes: %s/%s: %s", repo.Name(), branchName, kvPath) 53 | 54 | data, err := ioutil.ReadFile(fullpath) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | p := &api.KVPair{ 60 | Key: kvPath, 61 | Value: data, 62 | } 63 | 64 | _, err = h.Put(p, nil) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | return nil 70 | } 71 | 72 | err := filepath.Walk(repo.Workdir(), pushFile) 73 | if err != nil { 74 | log.WithError(err).Debug("PUT branch error") 75 | } 76 | 77 | return nil 78 | } 79 | -------------------------------------------------------------------------------- /watcher/interval.go: -------------------------------------------------------------------------------- 1 | package watch 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/Cimpress-MCP/go-git2consul/repository" 8 | "gopkg.in/libgit2/git2go.v24" 9 | ) 10 | 11 | // Watch the repo by interval. This is called as a go routine since 12 | // ticker blocks 13 | func (w *Watcher) pollByInterval(repo *repository.Repository, wg *sync.WaitGroup) { 14 | defer wg.Done() 15 | 16 | hooks := repo.Config.Hooks 17 | interval := time.Second 18 | 19 | // Find polling hook 20 | for _, h := range hooks { 21 | if h.Type == "polling" { 22 | interval = h.Interval 23 | break 24 | } 25 | } 26 | 27 | // If no polling found, don't poll 28 | if interval == 0 { 29 | return 30 | } 31 | 32 | ticker := time.NewTicker(interval * time.Second) 33 | defer ticker.Stop() 34 | 35 | // Polling error should not stop polling by interval 36 | for { 37 | err := w.pollBranches(repo) 38 | if err != nil { 39 | w.ErrCh <- err 40 | } 41 | 42 | if w.once { 43 | return 44 | } 45 | 46 | select { 47 | case <-ticker.C: 48 | case <-w.RcvDoneCh: 49 | return 50 | } 51 | } 52 | } 53 | 54 | // Watch all branches of a repository 55 | func (w *Watcher) pollBranches(repo *repository.Repository) error { 56 | itr, err := repo.NewBranchIterator(git.BranchLocal) 57 | if err != nil { 58 | return err 59 | } 60 | defer itr.Free() 61 | 62 | var checkoutBranchFn = func(b *git.Branch, _ git.BranchType) error { 63 | branchName, err := b.Name() 64 | if err != nil { 65 | return err 66 | } 67 | analysis, err := repo.Pull(branchName) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | // If there is a change, send the repo RepoChangeCh 73 | switch { 74 | case analysis&git.MergeAnalysisUpToDate != 0: 75 | w.logger.Debugf("Up to date: %s/%s", repo.Name(), branchName) 76 | case analysis&git.MergeAnalysisNormal != 0, analysis&git.MergeAnalysisFastForward != 0: 77 | w.logger.Infof("Changed: %s/%s", repo.Name(), branchName) 78 | w.RepoChangeCh <- repo 79 | } 80 | 81 | return nil 82 | } 83 | 84 | err = itr.ForEach(checkoutBranchFn) 85 | if err != nil && !git.IsErrorCode(err, git.ErrIterOver) { 86 | return err 87 | } 88 | 89 | return nil 90 | } 91 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # TODO 2 | 3 | ## Initial version requirements: 4 | * [x] Better error handling of goroutines through errCh 5 | * [x] Possible usage of a runner to abstract git operations from the consul package 6 | * [x] Update on KV should be for modified and deleted files only 7 | * [x] Switch from godep to glide 8 | * [x] Switch to apex/log 9 | * [x] Webhook polling 10 | * [x] GitHub 11 | * [x] Stash 12 | * [x] Bitbucket 13 | * [x] Gitlab 14 | * [x] Accept consul configuration for the client 15 | * [x] Add -once flag to run git2consul once 16 | * [ ] Better CD/CI pipeline 17 | * [ ] Cross-platform builds 18 | * [ ] Travis/appveyor 19 | 20 | ## Bugs/Issues: 21 | * [x] Need to update diffs on the KV side 22 | * [x] This includes only updating changed files 23 | * [x] Delete untracked files 24 | * [x] If repositories slice is empty, stop the program 25 | * [x] Directory check has to check if it's a repository first 26 | * [x] Runner, and watchers need a Stop() to handle cleanup better 27 | * [x] Handle DoneCh better on both the watcher and runner 28 | * [x] Handle initial load state better 29 | * [x] Watcher should handle initial changes from load state 30 | 31 | ## Error handling: 32 | * [x] Better error handling on LoadRepos() 33 | * [x] Bad configuration should be ignored 34 | * [ ] Handle repository error with git reset or re-clone 35 | 36 | ## Repo delta KV handling: 37 | * [x] On added, modified: PUT KV 38 | * [x] On delete: DEL KV 39 | * [x] On rename: DEL old KV followed by PUT new KV 40 | 41 | ## Test coverage 42 | * [x] repository 43 | * [x] New 44 | * [x] Clone 45 | * [x] Load 46 | * [x] Pull 47 | * [x] Diff 48 | * [x] Ref 49 | * [x] Checkout 50 | * [x] config 51 | * [x] Load 52 | * [ ] runner 53 | * [ ] New 54 | * [ ] watcher 55 | * [ ] Watcher 56 | * [x] Interval 57 | * [ ] Webhook 58 | * [ ] kv 59 | * [ ] Handler 60 | * [ ] Branch 61 | * [ ] KV 62 | * [ ] InitHandler 63 | * [ ] UpdateHandler 64 | 65 | Test suite enhancement: 66 | * [ ] git-init on repo should be done on init() 67 | * [ ] Setup and teardown for each test during 68 | * [ ] Setup resets "remote" repo to initial commit 69 | * [ ] Teardown cleans local store 70 | 71 | * Instead of testutil, we can use mocks to set up a mock repository.Repository object 72 | -------------------------------------------------------------------------------- /repository/pull.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "fmt" 5 | 6 | "gopkg.in/libgit2/git2go.v24" 7 | ) 8 | 9 | // Pull a repository branch, which is equivalent to a fetch and merge 10 | func (r *Repository) Pull(branchName string) (git.MergeAnalysis, error) { 11 | r.Lock() 12 | defer r.Unlock() 13 | 14 | origin, err := r.Remotes.Lookup("origin") 15 | if err != nil { 16 | return 0, err 17 | } 18 | defer origin.Free() 19 | 20 | rawLocalBranchRef := fmt.Sprintf("refs/heads/%s", branchName) 21 | 22 | // Fetch 23 | err = origin.Fetch([]string{rawLocalBranchRef}, nil, "") 24 | if err != nil { 25 | return 0, err 26 | } 27 | 28 | rawRemoteBranchRef := fmt.Sprintf("refs/remotes/origin/%s", branchName) 29 | 30 | remoteBranchRef, err := r.References.Lookup(rawRemoteBranchRef) 31 | if err != nil { 32 | return 0, err 33 | } 34 | 35 | // If the ref on the branch doesn't exist locally, create it 36 | // This also creates the branch 37 | _, err = r.References.Lookup(rawLocalBranchRef) 38 | if err != nil { 39 | _, err = r.References.Create(rawLocalBranchRef, remoteBranchRef.Target(), true, "") 40 | if err != nil { 41 | return 0, err 42 | } 43 | } 44 | 45 | // Change the HEAD to current branch and checkout 46 | err = r.SetHead(rawLocalBranchRef) 47 | if err != nil { 48 | return 0, err 49 | } 50 | err = r.CheckoutHead(&git.CheckoutOpts{ 51 | Strategy: git.CheckoutForce, 52 | }) 53 | if err != nil { 54 | return 0, err 55 | } 56 | 57 | head, err := r.Head() 58 | if err != nil { 59 | return 0, err 60 | } 61 | 62 | // Create annotated commit 63 | annotatedCommit, err := r.AnnotatedCommitFromRef(remoteBranchRef) 64 | if err != nil { 65 | return 0, err 66 | } 67 | 68 | // Merge analysis 69 | mergeHeads := []*git.AnnotatedCommit{annotatedCommit} 70 | analysis, _, err := r.MergeAnalysis(mergeHeads) 71 | if err != nil { 72 | return 0, err 73 | } 74 | 75 | // Action on analysis 76 | switch { 77 | case analysis&git.MergeAnalysisFastForward != 0, analysis&git.MergeAnalysisNormal != 0: 78 | if err := r.Merge(mergeHeads, nil, nil); err != nil { 79 | return 0, err 80 | } 81 | 82 | // Update refs on heads (local) from remotes 83 | if _, err = head.SetTarget(remoteBranchRef.Target(), ""); err != nil { 84 | return analysis, err 85 | } 86 | } 87 | 88 | defer head.Free() 89 | defer r.StateCleanup() 90 | return analysis, nil 91 | } 92 | -------------------------------------------------------------------------------- /watcher/watcher.go: -------------------------------------------------------------------------------- 1 | package watch 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/Cimpress-MCP/go-git2consul/config" 7 | "github.com/Cimpress-MCP/go-git2consul/repository" 8 | "github.com/apex/log" 9 | ) 10 | 11 | // Watcher is used to keep track of changes of the repositories 12 | type Watcher struct { 13 | sync.Mutex 14 | logger *log.Entry 15 | 16 | Repositories []*repository.Repository 17 | 18 | RepoChangeCh chan *repository.Repository 19 | ErrCh chan error 20 | RcvDoneCh chan struct{} 21 | SndDoneCh chan struct{} 22 | 23 | hookSvr *config.HookSvrConfig 24 | once bool 25 | } 26 | 27 | // New create a new watcher, passing in the the repositories, webhook 28 | // listener config, and optional once flag 29 | func New(repos []*repository.Repository, hookSvr *config.HookSvrConfig, once bool) *Watcher { 30 | repoChangeCh := make(chan *repository.Repository, len(repos)) 31 | logger := log.WithField("caller", "watcher") 32 | 33 | return &Watcher{ 34 | Repositories: repos, 35 | RepoChangeCh: repoChangeCh, 36 | ErrCh: make(chan error), 37 | RcvDoneCh: make(chan struct{}, 1), 38 | SndDoneCh: make(chan struct{}, 1), 39 | logger: logger, 40 | hookSvr: hookSvr, 41 | once: once, 42 | } 43 | } 44 | 45 | // Watch repositories available to the watcher 46 | func (w *Watcher) Watch() { 47 | defer close(w.SndDoneCh) 48 | 49 | // Pass repositories to RepoChangeCh for initial update to the KV 50 | for _, repo := range w.Repositories { 51 | w.RepoChangeCh <- repo 52 | } 53 | 54 | // WaitGroup size is equal to number of interval goroutine plus webhook goroutine 55 | var wg sync.WaitGroup 56 | wg.Add(len(w.Repositories) + 1) 57 | 58 | for _, repo := range w.Repositories { 59 | go w.pollByInterval(repo, &wg) 60 | } 61 | 62 | go w.pollByWebhook(&wg) 63 | 64 | go func() { 65 | wg.Wait() 66 | // Only exit if it's -once, otherwise there might be webhook polling 67 | if w.once { 68 | w.Stop() 69 | return 70 | } 71 | }() 72 | 73 | for { 74 | select { 75 | case err := <-w.ErrCh: 76 | log.WithError(err).Error("Watcher error") 77 | case <-w.RcvDoneCh: 78 | w.logger.Info("Received finish") 79 | wg.Wait() 80 | return 81 | } 82 | } 83 | } 84 | 85 | // Stop watching for changes. It will stop interval and webhook polling 86 | func (w *Watcher) Stop() { 87 | w.logger.Info("Stopping watcher...") 88 | close(w.RcvDoneCh) 89 | } 90 | -------------------------------------------------------------------------------- /repository/load_test.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/Cimpress-MCP/go-git2consul/config/mock" 10 | "github.com/Cimpress-MCP/go-git2consul/testutil" 11 | "github.com/apex/log" 12 | "github.com/apex/log/handlers/discard" 13 | "gopkg.in/libgit2/git2go.v24" 14 | ) 15 | 16 | func init() { 17 | log.SetHandler(discard.New()) 18 | } 19 | 20 | func TestLoadRepos(t *testing.T) { 21 | gitRepo, cleanup := testutil.GitInitTestRepo(t) 22 | defer cleanup() 23 | 24 | cfg := mock.Config(gitRepo.Workdir()) 25 | 26 | repos, err := LoadRepos(cfg) 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | 31 | // Cleanup cloning 32 | defer func() { 33 | for _, repo := range repos { 34 | os.RemoveAll(repo.Workdir()) 35 | } 36 | }() 37 | } 38 | 39 | func TestLoadRepos_existingDir(t *testing.T) { 40 | bareDir, err := ioutil.TempDir("", "bare-dir") 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | cfg := mock.Config(bareDir) 46 | 47 | _, err = LoadRepos(cfg) 48 | if err == nil { 49 | t.Fatal("Expected failure for existing repository") 50 | } 51 | 52 | // Cleanup 53 | defer func() { 54 | os.RemoveAll(bareDir) 55 | }() 56 | } 57 | 58 | func TestLoadRepos_invalidRepo(t *testing.T) { 59 | cfg := mock.Config("bogus-url") 60 | 61 | _, err := LoadRepos(cfg) 62 | if err == nil { 63 | t.Fatal("Expected failure for invalid repository url") 64 | } 65 | } 66 | 67 | func TestLoadRepos_existingRepo(t *testing.T) { 68 | gitRepo, cleanup := testutil.GitInitTestRepo(t) 69 | defer cleanup() 70 | 71 | cfg := mock.Config(gitRepo.Workdir()) 72 | localRepoPath := filepath.Join(cfg.LocalStore, cfg.Repos[0].Name) 73 | 74 | // Init a repo in the local store, with same name are the "remote" 75 | err := os.Mkdir(localRepoPath, 0755) 76 | if err != nil { 77 | t.Fatal(err) 78 | } 79 | 80 | repo, err := git.InitRepository(localRepoPath, false) 81 | if err != nil { 82 | t.Fatal(err) 83 | } 84 | 85 | _, err = repo.Remotes.Create("origin", "/foo/bar") 86 | if err != nil { 87 | t.Fatal(err) 88 | } 89 | 90 | repos, err := LoadRepos(cfg) 91 | if err != nil { 92 | t.Fatal(err) 93 | } 94 | 95 | // Cleanup cloning 96 | defer func() { 97 | for _, repo := range repos { 98 | os.RemoveAll(repo.Workdir()) 99 | } 100 | }() 101 | } 102 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | 10 | "github.com/Cimpress-MCP/go-git2consul/config" 11 | "github.com/Cimpress-MCP/go-git2consul/runner" 12 | "github.com/apex/log" 13 | "github.com/apex/log/handlers/text" 14 | ) 15 | 16 | // Exit code represented as int values for particular errors. 17 | const ( 18 | ExitCodeError = 10 + iota 19 | ExitCodeFlagError 20 | ExitCodeConfigError 21 | 22 | ExitCodeOk int = 0 23 | ) 24 | 25 | func main() { 26 | var filename string 27 | var version bool 28 | var debug bool 29 | var once bool 30 | 31 | flag.StringVar(&filename, "config", "", "path to config file") 32 | flag.BoolVar(&version, "version", false, "show version") 33 | flag.BoolVar(&debug, "debug", false, "enable debugging mode") 34 | flag.BoolVar(&once, "once", false, "run git2consul once and exit") 35 | flag.Parse() 36 | 37 | if debug { 38 | log.SetLevel(log.DebugLevel) 39 | } 40 | 41 | if version { 42 | fmt.Println("git2consul:") 43 | fmt.Printf(" %-9s%s\n", "Version:", Version) 44 | if GitCommit != "" { 45 | fmt.Printf(" %-9s%s\n", "Build:", GitCommit) 46 | } 47 | return 48 | } 49 | 50 | // TODO: Accept other logger inputs 51 | log.SetHandler(text.New(os.Stderr)) 52 | 53 | log.Infof("Starting git2consul version: %s", Version) 54 | 55 | if len(filename) == 0 { 56 | log.Error("No configuration file provided") 57 | os.Exit(ExitCodeFlagError) 58 | } 59 | 60 | // Load configuration from file 61 | cfg, err := config.Load(filename) 62 | if err != nil { 63 | log.Errorf("(config): %s", err) 64 | os.Exit(ExitCodeConfigError) 65 | } 66 | 67 | runner, err := runner.NewRunner(cfg, once) 68 | if err != nil { 69 | log.Errorf("(runner): %s", err) 70 | os.Exit(ExitCodeConfigError) 71 | } 72 | go runner.Start() 73 | 74 | signalCh := make(chan os.Signal, 1) 75 | signal.Notify(signalCh, 76 | syscall.SIGHUP, 77 | syscall.SIGINT, 78 | syscall.SIGTERM, 79 | syscall.SIGQUIT, 80 | ) 81 | 82 | for { 83 | select { 84 | case err := <-runner.ErrCh: 85 | log.WithError(err).Error("Runner error") 86 | os.Exit(ExitCodeError) 87 | case <-runner.SndDoneCh: // Used for cases like -once, where program is not terminated by interrupt 88 | log.Info("Terminating git2consul") 89 | os.Exit(ExitCodeOk) 90 | case <-signalCh: 91 | log.Info("Received interrupt. Cleaning up...") 92 | runner.Stop() 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /runner/runner.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/Cimpress-MCP/go-git2consul/config" 7 | "github.com/Cimpress-MCP/go-git2consul/kv" 8 | "github.com/Cimpress-MCP/go-git2consul/repository" 9 | "github.com/Cimpress-MCP/go-git2consul/watcher" 10 | "github.com/apex/log" 11 | ) 12 | 13 | // Runner is used to initialize a watcher and kvHandler 14 | type Runner struct { 15 | logger *log.Entry 16 | ErrCh chan error 17 | 18 | // Channel that receives done signal 19 | RcvDoneCh chan struct{} 20 | 21 | // Channel that sends done signal 22 | SndDoneCh chan struct{} 23 | 24 | once bool 25 | 26 | kvHandler *kv.KVHandler 27 | 28 | watcher *watch.Watcher 29 | } 30 | 31 | // NewRunner creates a new runner instance 32 | func NewRunner(config *config.Config, once bool) (*Runner, error) { 33 | logger := log.WithField("caller", "runner") 34 | 35 | // Create repos from configuration 36 | repos, err := repository.LoadRepos(config) 37 | if err != nil { 38 | return nil, fmt.Errorf("Cannot load repositories from configuration: %s", err) 39 | } 40 | 41 | // Create watcher to watch for repo changes 42 | watcher := watch.New(repos, config.HookSvr, once) 43 | 44 | // Create the handler 45 | handler, err := kv.New(config.Consul) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | runner := &Runner{ 51 | logger: logger, 52 | ErrCh: make(chan error), 53 | RcvDoneCh: make(chan struct{}, 1), 54 | SndDoneCh: make(chan struct{}, 1), 55 | once: once, 56 | kvHandler: handler, 57 | watcher: watcher, 58 | } 59 | 60 | return runner, nil 61 | } 62 | 63 | // Start the runner 64 | func (r *Runner) Start() { 65 | defer close(r.SndDoneCh) 66 | 67 | go r.watcher.Watch() 68 | 69 | for { 70 | select { 71 | case repo := <-r.watcher.RepoChangeCh: 72 | // Handle change, and return if error on handler 73 | err := r.kvHandler.HandleUpdate(repo) 74 | if err != nil { 75 | r.ErrCh <- err 76 | return 77 | } 78 | case <-r.watcher.SndDoneCh: // This triggers when watcher gets an error that causes termination 79 | r.logger.Info("Watcher received finish") 80 | return 81 | case <-r.RcvDoneCh: 82 | r.logger.Info("Received finish") 83 | return 84 | } 85 | } 86 | } 87 | 88 | // Stop the runner, cleaning up any routines that it's running. In this case, it will stop 89 | // the watcher before closing DoneCh 90 | func (r *Runner) Stop() { 91 | r.logger.Info("Stopping runner...") 92 | r.watcher.Stop() 93 | <-r.watcher.SndDoneCh // NOTE: Might need a timeout to prevent blocking forever 94 | close(r.RcvDoneCh) 95 | } 96 | -------------------------------------------------------------------------------- /repository/repository.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/Cimpress-MCP/go-git2consul/config" 11 | "gopkg.in/libgit2/git2go.v24" 12 | ) 13 | 14 | // Repository is used to hold the git repository object and it's configuration 15 | type Repository struct { 16 | sync.Mutex 17 | 18 | *git.Repository 19 | Config *config.Repo 20 | } 21 | 22 | // Status codes for Repository object creation 23 | const ( 24 | RepositoryError = iota // Unused, it will always get returned with an err 25 | RepositoryCloned 26 | RepositoryOpened 27 | ) 28 | 29 | // Name returns the repository name 30 | func (r *Repository) Name() string { 31 | return r.Config.Name 32 | } 33 | 34 | // Branch returns the branch name 35 | func (r *Repository) Branch() string { 36 | head, err := r.Head() 37 | if err != nil { 38 | return "" 39 | } 40 | bn, err := head.Branch().Name() 41 | if err != nil { 42 | return "" 43 | } 44 | 45 | return bn 46 | } 47 | 48 | // New is used to construct a new repository object from the configuration 49 | func New(repoBasePath string, repoConfig *config.Repo) (*Repository, int, error) { 50 | repoPath := filepath.Join(repoBasePath, repoConfig.Name) 51 | 52 | r := &Repository{ 53 | Repository: &git.Repository{}, 54 | Config: repoConfig, 55 | } 56 | 57 | state, err := r.init(repoPath) 58 | if err != nil { 59 | return nil, RepositoryError, err 60 | } 61 | 62 | if r.Repository == nil { 63 | return nil, RepositoryError, fmt.Errorf("Could not find git repostory") 64 | } 65 | 66 | return r, state, nil 67 | } 68 | 69 | // Initialize git.Repository object by opening the repostory or cloning from 70 | // the source URL. It does not handle purging existing file or directory 71 | // with the same path 72 | func (r *Repository) init(repoPath string) (int, error) { 73 | gitRepo, err := git.OpenRepository(repoPath) 74 | if err != nil || gitRepo == nil { 75 | err := r.Clone(repoPath) 76 | if err != nil { 77 | return RepositoryError, err 78 | } 79 | return RepositoryCloned, nil 80 | } 81 | 82 | // If remote URL are not the same, it will purge local copy and re-clone 83 | if r.mismatchRemoteUrl(gitRepo) { 84 | os.RemoveAll(gitRepo.Workdir()) 85 | err := r.Clone(repoPath) 86 | if err != nil { 87 | return RepositoryError, err 88 | } 89 | return RepositoryCloned, nil 90 | } 91 | 92 | r.Repository = gitRepo 93 | 94 | return RepositoryOpened, nil 95 | } 96 | 97 | func (r *Repository) mismatchRemoteUrl(gitRepo *git.Repository) bool { 98 | rm, err := gitRepo.Remotes.Lookup("origin") 99 | if err != nil { 100 | return true 101 | } 102 | 103 | if strings.Compare(rm.Url(), r.Config.Url) != 0 { 104 | return true 105 | } 106 | 107 | return false 108 | } 109 | -------------------------------------------------------------------------------- /config/load.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "time" 9 | 10 | "github.com/apex/log" 11 | "github.com/hashicorp/consul/api" 12 | ) 13 | 14 | // Load maps the configuration provided from a file to a Configuration object 15 | func Load(file string) (*Config, error) { 16 | // log context 17 | logger := log.WithFields(log.Fields{ 18 | "caller": "config", 19 | }) 20 | 21 | content, err := ioutil.ReadFile(file) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | // Create Config object pointer and unmashal JSON into it 27 | config := &Config{ 28 | Consul: &ConsulConfig{}, 29 | HookSvr: &HookSvrConfig{}, 30 | } 31 | err = json.Unmarshal(content, config) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | logger.Info("Setting configuration with sane defaults") 37 | config.setDefaultConfig() 38 | config.setDefaultConsulConfig() 39 | 40 | err = config.checkConfig() 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | jsonConfig, err := json.Marshal(config) 46 | if err != nil { 47 | return nil, err 48 | } 49 | logger.Debugf("Using configuration: %s", jsonConfig) 50 | 51 | return config, nil 52 | } 53 | 54 | // Check for the validitiy of the configuration file 55 | func (c *Config) checkConfig() error { 56 | for _, repo := range c.Repos { 57 | // Check on name 58 | if len(repo.Name) == 0 { 59 | return fmt.Errorf("Repository array object missing \"name\" value") 60 | } 61 | 62 | // Check on Url 63 | if len(repo.Url) == 0 { 64 | return fmt.Errorf("%s does no have a repository URL", repo.Name) 65 | } 66 | 67 | // Check on hooks 68 | for _, hook := range repo.Hooks { 69 | if hook.Type != "polling" && hook.Type != "webhook" { 70 | return fmt.Errorf("Invalid hook type: %s", hook.Type) 71 | } 72 | 73 | if hook.Type == "polling" && hook.Interval <= 0 { 74 | return fmt.Errorf("Invalid interval: %s. Hook interval must be greater than zero", hook.Interval) 75 | } 76 | } 77 | } 78 | 79 | return nil 80 | } 81 | 82 | // Return a configuration with sane defaults 83 | func (c *Config) setDefaultConfig() { 84 | 85 | // Set the default cache store to be the OS' temp dir 86 | if len(c.LocalStore) == 0 { 87 | c.LocalStore = os.TempDir() 88 | } 89 | 90 | // Set the default webhook port 91 | if c.HookSvr.Port == 0 { 92 | c.HookSvr.Port = 9000 93 | } 94 | 95 | //For each repo, set default branch and hook 96 | for _, repo := range c.Repos { 97 | branch := []string{"master"} 98 | // If there are no branches, set it to master 99 | if len(repo.Branches) == 0 { 100 | repo.Branches = branch 101 | } 102 | 103 | // If there are no hooks, set a 60s polling hook 104 | if len(repo.Hooks) == 0 { 105 | hook := &Hook{ 106 | Type: "polling", 107 | Interval: 60 * time.Second, 108 | } 109 | 110 | repo.Hooks = append(repo.Hooks, hook) 111 | } 112 | } 113 | } 114 | 115 | // This is to return default values so that the returned JSON is correctly populated 116 | func (c *Config) setDefaultConsulConfig() { 117 | defConfig := api.DefaultConfig() 118 | 119 | if c.Consul.Address == "" { 120 | c.Consul.Address = defConfig.Address 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /kv/init.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "github.com/Cimpress-MCP/go-git2consul/repository" 5 | "gopkg.in/libgit2/git2go.v24" 6 | ) 7 | 8 | // HandleInit handles initial fetching of the KV on start 9 | func (h *KVHandler) HandleInit(repos []*repository.Repository) error { 10 | for _, repo := range repos { 11 | err := h.handleRepoInit(repo) 12 | if err != nil { 13 | return err 14 | } 15 | } 16 | 17 | return nil 18 | } 19 | 20 | // Handles differences on all branches of a repository, comparing the ref 21 | // of the branch against the one in the KV 22 | func (h *KVHandler) handleRepoInit(repo *repository.Repository) error { 23 | repo.Lock() 24 | defer repo.Unlock() 25 | 26 | itr, err := repo.NewReferenceIterator() 27 | if err != nil { 28 | return err 29 | } 30 | defer itr.Free() 31 | 32 | // Handle all local refs 33 | for { 34 | ref, err := itr.Next() 35 | if err != nil { 36 | break 37 | } 38 | 39 | b, err := ref.Branch().Name() 40 | if err != nil { 41 | return err 42 | } 43 | 44 | // Get only local refs 45 | if ref.IsRemote() == false { 46 | h.logger.Infof("KV GET ref: %s/%s", repo.Name(), b) 47 | kvRef, err := h.getKVRef(repo, b) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | localRef := ref.Target().String() 53 | 54 | if len(kvRef) == 0 { 55 | // There is no ref in the KV, push the entire branch 56 | h.logger.Infof("KV PUT changes: %s/%s", repo.Name(), b) 57 | h.putBranch(repo, ref.Branch()) 58 | 59 | h.logger.Infof("KV PUT ref: %s/%s", repo.Name(), b) 60 | h.putKVRef(repo, b) 61 | } else if kvRef != localRef { 62 | // Check if the ref belongs to that repo 63 | err := repo.CheckRef(kvRef) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | // Handle modified and deleted files 69 | deltas, err := repo.DiffStatus(kvRef) 70 | if err != nil { 71 | return err 72 | } 73 | h.handleDeltas(repo, deltas) 74 | 75 | err = h.putKVRef(repo, b) 76 | if err != nil { 77 | return err 78 | } 79 | h.logger.Debugf("KV PUT ref: %s/%s", repo.Name(), b) 80 | } 81 | } 82 | } 83 | 84 | return nil 85 | } 86 | 87 | // Helper function that handles deltas 88 | func (h *KVHandler) handleDeltas(repo *repository.Repository, deltas []git.DiffDelta) error { 89 | // Handle modified and deleted files 90 | for _, d := range deltas { 91 | switch d.Status { 92 | case git.DeltaRenamed: 93 | h.logger.Debugf("Detected renamed file: %s", d.NewFile.Path) 94 | h.logger.Infof("KV DEL %s/%s/%s", repo.Name(), repo.Branch(), d.OldFile.Path) 95 | err := h.deleteKV(repo, d.OldFile.Path) 96 | if err != nil { 97 | return err 98 | } 99 | h.logger.Infof("KV PUT %s/%s/%s", repo.Name(), repo.Branch(), d.NewFile.Path) 100 | err = h.putKV(repo, d.NewFile.Path) 101 | if err != nil { 102 | return err 103 | } 104 | case git.DeltaAdded, git.DeltaModified: 105 | h.logger.Debugf("Detected added/modified file: %s", d.NewFile.Path) 106 | h.logger.Infof("KV PUT %s/%s/%s", repo.Name(), repo.Branch(), d.NewFile.Path) 107 | err := h.putKV(repo, d.NewFile.Path) 108 | if err != nil { 109 | return err 110 | } 111 | case git.DeltaDeleted: 112 | h.logger.Debugf("Detected deleted file: %s", d.OldFile.Path) 113 | h.logger.Infof("KV DEL %s/%s/%s", repo.Name(), repo.Branch(), d.OldFile.Path) 114 | err := h.deleteKV(repo, d.OldFile.Path) 115 | if err != nil { 116 | return err 117 | } 118 | } 119 | } 120 | 121 | return nil 122 | } 123 | -------------------------------------------------------------------------------- /testutil/git_repo.go: -------------------------------------------------------------------------------- 1 | // Package testutil takes care of initializing a local git repository for 2 | // testing. The 'remote' should match the one specified in config/mock. 3 | package testutil 4 | 5 | import ( 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | "path" 10 | "path/filepath" 11 | "runtime" 12 | "strings" 13 | "testing" 14 | "time" 15 | 16 | "gopkg.in/libgit2/git2go.v24" 17 | ) 18 | 19 | var testRepo *git.Repository 20 | 21 | // Return the test-fixtures path in testutil 22 | func fixturesRepo(t *testing.T) string { 23 | _, filename, _, ok := runtime.Caller(0) 24 | if !ok { 25 | t.Fatal("Cannot find path") 26 | } 27 | 28 | testutilPath := filepath.Dir(filename) 29 | return filepath.Join(testutilPath, "test-fixtures", "example") 30 | } 31 | 32 | func copyDir(srcPath string, dstPath string) error { 33 | // Copy fixtures into temporary path. filepath is the full path 34 | var copyFn = func(path string, info os.FileInfo, err error) error { 35 | currentFilePath := strings.TrimPrefix(path, srcPath) 36 | targetPath := filepath.Join(dstPath, currentFilePath) 37 | if info.IsDir() { 38 | if targetPath != dstPath { 39 | err := os.Mkdir(targetPath, 0755) 40 | if err != nil { 41 | return err 42 | } 43 | } 44 | } else { 45 | src, err := os.Open(path) 46 | if err != nil { 47 | return err 48 | } 49 | dst, err := os.Create(targetPath) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | _, err = io.Copy(dst, src) 55 | if err != nil { 56 | return err 57 | } 58 | } 59 | 60 | return nil 61 | } 62 | 63 | err := filepath.Walk(srcPath, copyFn) 64 | return err 65 | } 66 | 67 | // GitInitTestRepo coppies test-fixtures to os.TempDir() and performs a 68 | // git-init on directory. 69 | func GitInitTestRepo(t *testing.T) (*git.Repository, func()) { 70 | fixtureRepo := fixturesRepo(t) 71 | repoPath, err := ioutil.TempDir("", "git2consul-test-remote") 72 | if err != nil { 73 | t.Fatal(err) 74 | } 75 | 76 | err = copyDir(fixtureRepo, repoPath) 77 | if err != nil { 78 | t.Fatal(err) 79 | } 80 | 81 | // Init repo 82 | repo, err := git.InitRepository(repoPath, false) 83 | if err != nil { 84 | t.Fatal(err) 85 | } 86 | 87 | // Add files to index 88 | idx, err := repo.Index() 89 | if err != nil { 90 | t.Fatal(err) 91 | } 92 | err = idx.AddAll([]string{}, git.IndexAddDefault, nil) 93 | if err != nil { 94 | t.Fatal(err) 95 | } 96 | err = idx.Write() 97 | if err != nil { 98 | t.Fatal(err) 99 | } 100 | 101 | treeId, err := idx.WriteTree() 102 | if err != nil { 103 | t.Fatal(err) 104 | } 105 | 106 | tree, err := repo.LookupTree(treeId) 107 | if err != nil { 108 | t.Fatal(err) 109 | } 110 | 111 | // Initial commit 112 | sig := &git.Signature{ 113 | Name: "Test Example", 114 | Email: "tes@example.com", 115 | When: time.Date(2016, 01, 01, 12, 00, 00, 0, time.UTC), 116 | } 117 | 118 | repo.CreateCommit("HEAD", sig, sig, "Initial commit", tree) 119 | testRepo = repo 120 | 121 | // Cleanup function that removes the repository directory 122 | var cleanup = func() { 123 | os.RemoveAll(repoPath) 124 | } 125 | 126 | return repo, cleanup 127 | } 128 | 129 | // GitCommitTestRepo performs a commit on the test repository, and returns 130 | // its Oid as well as a cleanup function to revert those changes. 131 | func GitCommitTestRepo(t *testing.T) (*git.Oid, func()) { 132 | // Save commmit ref for reset later 133 | h, err := testRepo.Head() 134 | if err != nil { 135 | t.Fatal(err) 136 | } 137 | 138 | obj, err := testRepo.Lookup(h.Target()) 139 | if err != nil { 140 | t.Fatal(err) 141 | } 142 | 143 | initialCommit, err := obj.AsCommit() 144 | if err != nil { 145 | t.Fatal(err) 146 | } 147 | 148 | date := []byte(time.Now().String()) 149 | file := path.Join(testRepo.Workdir(), "foo") 150 | err = ioutil.WriteFile(file, date, 0755) 151 | if err != nil { 152 | t.Fatal(err) 153 | } 154 | 155 | // Commit changes 156 | idx, err := testRepo.Index() 157 | if err != nil { 158 | t.Fatal(err) 159 | } 160 | 161 | err = idx.AddByPath("foo") 162 | if err != nil { 163 | t.Fatal(err) 164 | } 165 | 166 | treeId, err := idx.WriteTree() 167 | 168 | tree, err := testRepo.LookupTree(treeId) 169 | if err != nil { 170 | t.Fatal(err) 171 | } 172 | 173 | h, err = testRepo.Head() 174 | if err != nil { 175 | t.Fatal(err) 176 | } 177 | 178 | commit, err := testRepo.LookupCommit(h.Target()) 179 | if err != nil { 180 | t.Fatal(err) 181 | } 182 | 183 | sig := &git.Signature{ 184 | Name: "Test Example", 185 | Email: "test@example.com", 186 | When: time.Date(2016, 01, 01, 12, 00, 00, 0, time.UTC), 187 | } 188 | 189 | oid, err := testRepo.CreateCommit("HEAD", sig, sig, "Update commit", tree, commit) 190 | if err != nil { 191 | t.Fatal(err) 192 | } 193 | 194 | // Undo commit 195 | var cleanup = func() { 196 | testRepo.ResetToCommit(initialCommit, git.ResetHard, &git.CheckoutOpts{ 197 | Strategy: git.CheckoutForce, 198 | }) 199 | 200 | testRepo.StateCleanup() 201 | } 202 | 203 | return oid, cleanup 204 | } 205 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-git2consul 2 | 3 | [![Go Report Card](https://goreportcard.com/badge/github.com/Cimpress-MCP/go-git2consul)][goreport] 4 | 5 | [goreport]: https://goreportcard.com/report/github.com/Cimpress-MCP/go-git2consul 6 | 7 | ***NOTE: go-git2consul is experimental and still under development, and therefore should not be used in production!*** 8 | 9 | go-git2consul is a port of [git2consul](https://github.com/Cimpress-MCP/git2consul), which had great success and adoption. go-git2consul takes on the same basic principles as its predecessor, and attempts to improve upon some of its feature sets as well as add new ones. There are a few advantages to go-git2consul, including, but is not limited to, the use of the official Consul API and the removal of runtime dependencies such as node and git. 10 | 11 | Configuration on go-git2consul is sourced locally instead of it being fetched from the KV. This provides better isolation in cases where multiple instances of git2consul are running in order to provide high-availability, and addresses the issues mentioned in [Cimpress-MCP/git2consul#73](https://github.com/Cimpress-MCP/git2consul/issues/73). 12 | 13 | ## Configuration 14 | 15 | Configuration is provided with a JSON file and passed in via the `-config` flag. Repository configuration will take care of cloning the repository into `local_store`, but it will not be responsible for creating the actual `local_store` directory. Similarly, it is expected that there is no collision of directory or file that contains the same name as the repository name under `local_store`, or git2consul will exit with an error. If there is a git repository under a specified repo name, and the origin URL is different from the one provided in the configuration, it will be overwritten. 16 | 17 | ### Default configuration 18 | 19 | git2consul will attempt to use sane defaults for configuration. However, since git2consul needs to know which repository to pull from, minimal configuration is necessary. 20 | 21 | | Configuration | Required | Default Value | Available Values | Description 22 | |----------------------|----------|----------------|--------------------------------------------| ----------- 23 | | local_store | no | `os.TempDir()` | `string` | Local cache for git2consul to store its tracked repositories 24 | | webhook:address | no | | `string` | Webhook listener address that git2consul will be using 25 | | webhook:port | no | 9000 | `int` | Webhook listener port that git2consul will be using 26 | | repos:name | yes | | `string` | Name of the repository. This will match the webhook path, if any are enabled 27 | | repos:url | yes | | `string` | The URL of the repository 28 | | repos:branches | no | master | `string` | Tracking branches of the repository 29 | | repos:hooks:type | no | polling | polling, github, stash, bitbucket, gitlab | Type of hook to use to fetch changes on the repository 30 | | repos:hooks:interval | no | 60 | `int` | Interval, in seconds, to poll if polling is enabled 31 | | consul:address | no | 127.0.0.1:8500 | `string` | Consul address to connect to. It can be either the IP or FQDN with port included 32 | | consul:ssl | no | false | true, false | Whether to use HTTPS to communicate with Consul 33 | | consul:ssl_verify | no | false | true, false | Whether to verify certificates when connecting via SSL 34 | | consul:token | no | | `string` | Consul API Token 35 | 36 | ## Available command option flags 37 | 38 | ### `-config` 39 | The path to the configuration file. This flag is *required*. 40 | 41 | ### `-once` 42 | Runs git2consul once and exits. This essentially ignores webhook polling. 43 | 44 | ### `-version` 45 | Displays the version of git2consul and exits. All other commands are ignored. 46 | 47 | ## Webhooks 48 | 49 | Webhooks will be served from a single port, and different repositories will be given different endpoints according to their name 50 | 51 | Available endpoints: 52 | 53 | * `:/{repository}/github` 54 | * `:/{repository}/stash` 55 | * `:/{repository}/bitbucket` 56 | * `:/{repository}/gitlab` 57 | 58 | ## Future feature additions 59 | * File format backend 60 | * Support for source_root and mountpoint 61 | * Support for tags as branches 62 | * Support for Consul HTTP Basic Auth 63 | * Logger support for other handlers other than text 64 | * Auth support for webhooks banckends 65 | 66 | 67 | ## Development dependencies 68 | * Go 1.6 69 | * libgit2 v0.24.0 70 | * [glide](https://github.com/Masterminds/glide) 71 | 72 | 73 | *Influenced by these awesome tools: git2consul, consul-replicate, fabio* 74 | -------------------------------------------------------------------------------- /watcher/webhook.go: -------------------------------------------------------------------------------- 1 | package watch 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "sort" 9 | "sync" 10 | 11 | "github.com/gorilla/mux" 12 | "gopkg.in/libgit2/git2go.v24" 13 | ) 14 | 15 | // GithubPayload is the response from GitHub 16 | type GithubPayload struct { 17 | Ref string `json:"ref"` 18 | } 19 | 20 | // StashPayload is the response from Stash 21 | type StashPayload struct { 22 | RefChanges []struct { 23 | RefId string `json:"refId"` 24 | } `json:"refChanges"` 25 | } 26 | 27 | // BitbucketPayload is the response Bitbucket 28 | type BitbucketPayload struct { 29 | Push struct { 30 | Changes []struct { 31 | New struct { 32 | Name string `json:"name"` 33 | } `json:"new"` 34 | } `json:"changes"` 35 | } `json:"push"` 36 | } 37 | 38 | // GitLabPayload is the response from GitLab 39 | type GitLabPayload struct { 40 | Ref string `json:"ref"` 41 | } 42 | 43 | func (w *Watcher) pollByWebhook(wg *sync.WaitGroup) { 44 | defer wg.Done() 45 | 46 | if w.once { 47 | return 48 | } 49 | 50 | errCh := make(chan error, 1) 51 | // Passing errCh instead of w.ErrCh to better handle watcher termination 52 | // since the caller can't determine what type of error it receives from watcher 53 | go w.ListenAndServe(errCh) 54 | 55 | for { 56 | select { 57 | case err := <-errCh: 58 | w.ErrCh <- err 59 | close(w.RcvDoneCh) // Stop the watcher if there is a 60 | case <-w.RcvDoneCh: 61 | return 62 | } 63 | } 64 | } 65 | 66 | // ListenAndServe starts the listener server for hooks 67 | func (w *Watcher) ListenAndServe(errCh chan<- error) { 68 | r := mux.NewRouter() 69 | r.HandleFunc("/{repository}/github", w.githubHandler) 70 | r.HandleFunc("/{repository}/stash", w.stashHandler) 71 | r.HandleFunc("/{repository}/bitbucket", w.bitbucketHandler) 72 | r.HandleFunc("/{repository}/gitlab", w.gitlabHandler) 73 | 74 | addr := fmt.Sprintf("%s:%d", w.hookSvr.Address, w.hookSvr.Port) 75 | errCh <- http.ListenAndServe(addr, r) 76 | } 77 | 78 | // HTTP handler for github 79 | func (w *Watcher) githubHandler(rw http.ResponseWriter, rq *http.Request) { 80 | vars := mux.Vars(rq) 81 | repository := vars["repository"] 82 | 83 | eventType := rq.Header.Get("X-Github-Event") 84 | if eventType == "" { 85 | http.Error(rw, "Missing X-Github-Event header", http.StatusBadRequest) 86 | return 87 | } 88 | // Only process pusn events 89 | if eventType != "push" { 90 | return 91 | } 92 | 93 | body, err := ioutil.ReadAll(rq.Body) 94 | if err != nil { 95 | http.Error(rw, "Cannot read body", http.StatusInternalServerError) 96 | return 97 | } 98 | 99 | payload := &GithubPayload{} 100 | err = json.Unmarshal(body, payload) 101 | if err != nil { 102 | http.Error(rw, "Cannot unmarshal JSON", http.StatusInternalServerError) 103 | return 104 | } 105 | 106 | // Check the content 107 | ref := payload.Ref 108 | if len(ref) == 0 { 109 | http.Error(rw, "ref is empty", http.StatusInternalServerError) 110 | return 111 | } 112 | if len(ref) <= 11 || ref[:11] != "refs/heads/" { 113 | return 114 | } 115 | 116 | branchName := ref[11:] 117 | 118 | i := sort.Search(len(w.Repositories), func(i int) bool { 119 | return w.Repositories[i].Name() == repository 120 | }) 121 | 122 | // sort.Search could return last index if not found, so need to check once more 123 | if i == len(w.Repositories) || w.Repositories[i].Name() != repository { 124 | return 125 | } 126 | 127 | repo := w.Repositories[i] 128 | w.logger.WithField("repository", repo.Name()).Info("Received hook event from GitHub") 129 | analysis, err := repo.Pull(branchName) 130 | if err != nil { 131 | http.Error(rw, err.Error(), http.StatusInternalServerError) 132 | return 133 | } 134 | 135 | // If there is a change, send the repo RepoChangeCh 136 | switch { 137 | case analysis&git.MergeAnalysisUpToDate != 0: 138 | w.logger.Debugf("Up to date: %s/%s", repo.Name(), branchName) 139 | case analysis&git.MergeAnalysisNormal != 0, analysis&git.MergeAnalysisFastForward != 0: 140 | w.logger.Infof("Changed: %s/%s", repo.Name(), branchName) 141 | w.RepoChangeCh <- repo 142 | } 143 | } 144 | 145 | // HTTP handler for Stash 146 | func (w *Watcher) stashHandler(rw http.ResponseWriter, rq *http.Request) { 147 | vars := mux.Vars(rq) 148 | repository := vars["repository"] 149 | 150 | body, err := ioutil.ReadAll(rq.Body) 151 | if err != nil { 152 | http.Error(rw, "Cannot read body", http.StatusInternalServerError) 153 | return 154 | } 155 | 156 | payload := &StashPayload{} 157 | err = json.Unmarshal(body, payload) 158 | if err != nil { 159 | http.Error(rw, "Cannot unmarshal JSON", http.StatusInternalServerError) 160 | return 161 | } 162 | 163 | ref := payload.RefChanges[0].RefId 164 | 165 | if len(ref) == 0 { 166 | http.Error(rw, "ref is empty", http.StatusInternalServerError) 167 | return 168 | } 169 | if len(ref) <= 11 || ref[:11] != "refs/heads/" { 170 | return 171 | } 172 | 173 | branchName := ref[11:] 174 | 175 | i := sort.Search(len(w.Repositories), func(i int) bool { 176 | return w.Repositories[i].Name() == repository 177 | }) 178 | 179 | // sort.Search could return last index if not found, so need to check once more 180 | if i == len(w.Repositories) || w.Repositories[i].Name() != repository { 181 | return 182 | } 183 | 184 | repo := w.Repositories[i] 185 | w.logger.WithField("repository", repo.Name()).Info("Received hook event from Stash") 186 | analysis, err := repo.Pull(branchName) 187 | if err != nil { 188 | http.Error(rw, err.Error(), http.StatusInternalServerError) 189 | return 190 | } 191 | 192 | // If there is a change, send the repo RepoChangeCh 193 | switch { 194 | case analysis&git.MergeAnalysisUpToDate != 0: 195 | w.logger.Debugf("Up to date: %s/%s", repo.Name(), branchName) 196 | case analysis&git.MergeAnalysisNormal != 0, analysis&git.MergeAnalysisFastForward != 0: 197 | w.logger.Infof("Changed: %s/%s", repo.Name(), branchName) 198 | w.RepoChangeCh <- repo 199 | } 200 | } 201 | 202 | // HTTP handler for Bitbucket 203 | func (w *Watcher) bitbucketHandler(rw http.ResponseWriter, rq *http.Request) { 204 | vars := mux.Vars(rq) 205 | repository := vars["repository"] 206 | 207 | eventType := rq.Header.Get("X-Event-Key") 208 | if eventType == "" { 209 | http.Error(rw, "Missing X-Event-key header", http.StatusBadRequest) 210 | return 211 | } 212 | // Only process pusn events 213 | if eventType != "repo:push" { 214 | return 215 | } 216 | 217 | body, err := ioutil.ReadAll(rq.Body) 218 | if err != nil { 219 | http.Error(rw, "Cannot read body", http.StatusInternalServerError) 220 | return 221 | } 222 | 223 | payload := &BitbucketPayload{} 224 | err = json.Unmarshal(body, payload) 225 | if err != nil { 226 | http.Error(rw, "Cannot unmarshal JSON", http.StatusInternalServerError) 227 | return 228 | } 229 | 230 | // Check the content 231 | ref := payload.Push.Changes[0].New.Name 232 | if len(ref) == 0 { 233 | http.Error(rw, "ref is empty", http.StatusInternalServerError) 234 | return 235 | } 236 | if len(ref) <= 11 || ref[:11] != "refs/heads/" { 237 | return 238 | } 239 | 240 | branchName := ref[11:] 241 | 242 | i := sort.Search(len(w.Repositories), func(i int) bool { 243 | return w.Repositories[i].Name() == repository 244 | }) 245 | 246 | // sort.Search could return last index if not found, so need to check once more 247 | if i == len(w.Repositories) || w.Repositories[i].Name() != repository { 248 | return 249 | } 250 | 251 | repo := w.Repositories[i] 252 | w.logger.WithField("repository", repo.Name()).Info("Received hook event from Bitbucket") 253 | analysis, err := repo.Pull(branchName) 254 | if err != nil { 255 | http.Error(rw, err.Error(), http.StatusInternalServerError) 256 | return 257 | } 258 | 259 | // If there is a change, send the repo RepoChangeCh 260 | switch { 261 | case analysis&git.MergeAnalysisUpToDate != 0: 262 | w.logger.Debugf("Up to date: %s/%s", repo.Name(), branchName) 263 | case analysis&git.MergeAnalysisNormal != 0, analysis&git.MergeAnalysisFastForward != 0: 264 | w.logger.Infof("Changed: %s/%s", repo.Name(), branchName) 265 | w.RepoChangeCh <- repo 266 | } 267 | } 268 | 269 | func (w *Watcher) gitlabHandler(rw http.ResponseWriter, rq *http.Request) { 270 | vars := mux.Vars(rq) 271 | repository := vars["repository"] 272 | 273 | eventType := rq.Header.Get("X-Gitlab-Event") 274 | if eventType == "" { 275 | http.Error(rw, "Missing X-Gitlab-Event header", http.StatusBadRequest) 276 | return 277 | } 278 | // Only process pusn events 279 | if eventType != "Push Hook" { 280 | return 281 | } 282 | 283 | body, err := ioutil.ReadAll(rq.Body) 284 | if err != nil { 285 | http.Error(rw, "Cannot read body", http.StatusInternalServerError) 286 | return 287 | } 288 | 289 | payload := &GitLabPayload{} 290 | err = json.Unmarshal(body, payload) 291 | if err != nil { 292 | http.Error(rw, "Cannot unmarshal JSON", http.StatusInternalServerError) 293 | return 294 | } 295 | 296 | // Check the content 297 | ref := payload.Ref 298 | if len(ref) == 0 { 299 | http.Error(rw, "ref is empty", http.StatusInternalServerError) 300 | return 301 | } 302 | if len(ref) <= 11 || ref[:11] != "refs/heads/" { 303 | return 304 | } 305 | 306 | branchName := ref[11:] 307 | 308 | i := sort.Search(len(w.Repositories), func(i int) bool { 309 | return w.Repositories[i].Name() == repository 310 | }) 311 | 312 | // sort.Search could return last index if not found, so need to check once more 313 | if i == len(w.Repositories) || w.Repositories[i].Name() != repository { 314 | return 315 | } 316 | 317 | repo := w.Repositories[i] 318 | w.logger.WithField("repository", repo.Name()).Info("Received hook event from GitLab") 319 | analysis, err := repo.Pull(branchName) 320 | if err != nil { 321 | http.Error(rw, err.Error(), http.StatusInternalServerError) 322 | return 323 | } 324 | 325 | // If there is a change, send the repo RepoChangeCh 326 | switch { 327 | case analysis&git.MergeAnalysisUpToDate != 0: 328 | w.logger.Debugf("Up to date: %s/%s", repo.Name(), branchName) 329 | case analysis&git.MergeAnalysisNormal != 0, analysis&git.MergeAnalysisFastForward != 0: 330 | w.logger.Infof("Changed: %s/%s", repo.Name(), branchName) 331 | w.RepoChangeCh <- repo 332 | } 333 | } 334 | --------------------------------------------------------------------------------