├── Dockerfile ├── README.md ├── gitsplit ├── cache.go ├── config.go ├── reference_splitter_lite.go ├── remote.go ├── splitter.go ├── uri.go └── working_space.go ├── main.go └── utils ├── array.go ├── exec.go ├── file.go ├── hash.go └── pool.go /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine AS build 2 | 3 | RUN apk add --no-cache \ 4 | git 5 | 6 | RUN go get -d github.com/libgit2/git2go 7 | RUN cd $GOPATH/src/github.com/libgit2/git2go \ 8 | && git submodule update --init 9 | 10 | RUN apk add --no-cache \ 11 | make\ 12 | cmake \ 13 | g++ \ 14 | openssl-dev \ 15 | libssh2-dev 16 | 17 | RUN cd $GOPATH/src/github.com/libgit2/git2go \ 18 | && make install-static 19 | 20 | COPY . /go/src/github.com/jderusse/gitsplit/ 21 | 22 | RUN go get --tags "static" github.com/jderusse/gitsplit 23 | RUN go build --tags "static" -o gitsplit github.com/jderusse/gitsplit 24 | 25 | # ================================================== 26 | 27 | FROM alpine 28 | 29 | RUN apk add --no-cache \ 30 | git \ 31 | openssl \ 32 | openssh-client \ 33 | ca-certificates \ 34 | libssh2-dev 35 | 36 | COPY --from=build /go/gitsplit /bin/gitsplit 37 | 38 | WORKDIR /srv 39 | CMD ["gitsplit"] 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A Docker image with git and splitsh-lite 2 | 3 | See [the official site](https://github.com/splitsh/lite) for more information about splitsh. 4 | 5 | Demo available here [jderusse/test-split-a](https://github.com/jderusse/test-split-a). 6 | 7 | # Usage 8 | 9 | Include a `.gitsplit.yml` file in the root of your repository. 10 | This section provides a brief overview of the configuration file and split process. 11 | 12 | Use env variable to inject your credential and manage authentication. 13 | 14 | Example `.gitsplit.yml` configuration: 15 | 16 | ```yaml 17 | # Path to a cache directory Used to speed up the split over time by reusing git's objects 18 | cache_url: "/cache/gitsplit" 19 | # cache_url: "file:///cache/gitsplit" 20 | # cache_url: "https://${GH_TOKEN}@github.com/my_company/project-cache.git" 21 | # cache_url: "git@gitlab.com:my_company/project-cache.git" 22 | 23 | # Path to the repository to split (default = current path) 24 | # project_url: /home/me/workspace/another_project 25 | # project_url: ~/workspace/another_project 26 | # project_url: ../another_project 27 | # project_url: file://~/workspace/another_project 28 | # project_url: "https://${GH_TOKEN}@github.com/my_company/project.git" 29 | # project_url: "git@gitlab.com:my_company/project.git" 30 | 31 | # List of splits. 32 | splits: 33 | - prefix: "src/partA" 34 | target: "https://${GH_TOKEN}@github.com/my_company/project-partA.git" 35 | - prefix: "src/partB" 36 | target: 37 | # You can push the split to several repositories 38 | - "https://${GH_TOKEN}@github.com/my_company/project-partB.git" 39 | - "https://${GH_TOKEN}@github.com/my_company/project-partZ.git" 40 | - prefix: 41 | # You can use several prefix in the split 42 | - "src/subTree/PartC:" 43 | - "src/subTree/PartZ:lib/z" 44 | target: "https://${GH_TOKEN}@github.com/my_company/project-partC.git" 45 | 46 | # List of references to split (defined as regexp) 47 | origins: 48 | - ^master$ 49 | - ^develop$ 50 | - ^feature/ 51 | - ^v\d+\.\d+\.\d+$ 52 | ``` 53 | 54 | # Split your repo manualy 55 | 56 | With a github token: 57 | ``` 58 | $ docker run --rm -ti -e GH_TOKEN -v /cache:/cache/gitsplit -v $PWD:/srv jderusse/gitsplit 59 | ``` 60 | 61 | With ssh agent: 62 | ``` 63 | $ docker run --rm -ti -e SSH_AUTH_SOCK=/ssh-agent -v $SSH_AUTH_SOCK:/ssh-agent -v /cache:/cache/gitsplit -v $PWD:/srv jderusse/gitsplit 64 | ``` 65 | 66 | # Sample with drone.io 67 | 68 | Beware, the container have to push on your splited repository. 69 | It could be a security issue. Use environments variables as defined in the official documentation 70 | 71 | ```yaml 72 | # .gitsplit.yml 73 | cache_url: "/cache/gitsplit" 74 | splits: 75 | - prefix: "src/partA" 76 | target: "https://${GH_TOKEN}@github.com/my_company/project-partA.git" 77 | origins: 78 | - ^master$ 79 | ``` 80 | 81 | ```yaml 82 | # .drone.yml 83 | pipeline: 84 | split: 85 | image: jderusse/gitsplit 86 | pull: true 87 | volumes: 88 | # Share a cache mounted in the runner 89 | - /drone/cache/gitsplit:/cache/gitsplit 90 | 91 | # Use ssh key defined in the runner 92 | - /drone/env/gitsplit.ssh:/root/.ssh/ 93 | commands: 94 | # have to fetch remote branches 95 | - git fetch --prune --unshallow || git fetch --prune 96 | - gitsplit 97 | ``` 98 | 99 | # Sample with Travis CI 100 | 101 | Beware, the container have to push on your splited repository. 102 | It could be a security issue. Use environments variables as defined in the official documentation 103 | 104 | ```yaml 105 | # .gitsplit.yml 106 | cache_url: "/cache/gitsplit" 107 | splits: 108 | - prefix: "src/partA" 109 | target: "https://${GH_TOKEN}@github.com/my_company/project-partA.git" 110 | origins: 111 | - ^master$ 112 | ``` 113 | 114 | ```yaml 115 | # .travis.yml 116 | sudo: required 117 | services: 118 | - docker 119 | cache: 120 | directories: 121 | - /cache/gitsplit 122 | install: 123 | - docker pull jderusse/gitsplit 124 | 125 | # update local repository. Because travis fetch a shallow copy 126 | - git config remote.origin.fetch "+refs/*:refs/*" 127 | - git config remote.origin.mirror true 128 | - git fetch --prune --unshallow || git fetch --prune 129 | 130 | script: 131 | - docker run --rm -t -e GH_TOKEN -v /cache/gitsplit:/cache/gitsplit -v ${PWD}:/srv jderusse/gitsplit gitsplit --ref "${TRAVIS_BRANCH}" 132 | ``` 133 | 134 | # Sample with GitLab CI/CD 135 | 136 | Beware, the container have to push on your splited repository. 137 | It could be a security issue. Use environments variables as defined in the official documentation [GitLab SSH Deploy keys](https://docs.gitlab.com/ce/ssh/README.html#deploy-keys). 138 | 139 | Note: I highly recommend to use ssh instead of https because of the username/password or username/token. Deploy keys are much easier to use with GitLab 140 | 141 | ```yaml 142 | # .gitsplit.yml 143 | cache_url: "cache/gitsplit" 144 | splits: 145 | - prefix: "src/partA" 146 | target: "git@gitlab.com:my_company/project-partA.git" 147 | origins: 148 | - ^master$ 149 | ``` 150 | 151 | ```yaml 152 | # .gitlab-ci.yml with Docker runners 153 | stages: 154 | - split 155 | 156 | split: 157 | image: jderusse/gitsplit 158 | stage: split 159 | cache: 160 | key: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME" 161 | paths: 162 | - cache/gitsplit 163 | variables: 164 | GIT_STRATEGY: clone 165 | before_script: 166 | - eval $(ssh-agent -s) 167 | - mkdir -p ~/.ssh 168 | - chmod 700 ~/.ssh 169 | - echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config 170 | - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null 171 | - ssh-add -l 172 | script: 173 | - git config remote.origin.fetch "+refs/*:refs/*" 174 | - git config remote.origin.mirror true 175 | - git fetch --prune --unshallow || git fetch --prune 176 | - gitsplit --ref "${CI_COMMIT_REF_NAME}" 177 | ``` 178 | 179 | # Sample with Github Actions 180 | 181 | I recommend to not use [the checkout action](https://github.com/actions/checkout) because it will checkout only the last 182 | commit for performance purpose. In this example we clone the whole repository. Indeed, if you need to split only the 183 | last changes and you don't need to split tags, you can use it. 184 | 185 | Also we do not use the GH Action auto-generated token because it doesn't have the required permissions for gitsplit to 186 | work. You need to be able to write on all repositories you will split onto, you can create one by using following 187 | link: [https://github.com/settings/tokens/new?scopes=repo,workflow&description=gitsplit](https://github.com/settings/tokens/new?scopes=repo,workflow&description=gitsplit) 188 | 189 | If you want a fine grained token, you can create one with the following scopes: 190 | 191 | * https://github.com/settings/personal-access-tokens/new 192 | * all repo 193 | * repo > content > read write 194 | 195 | In this example I trigger the gitsplit on main branch push and on Github release (tags), feel free to make it your way ! 196 | 197 | ```yaml 198 | name: gitsplit 199 | on: 200 | push: 201 | branches: 202 | - main 203 | release: 204 | types: [published] 205 | 206 | jobs: 207 | gitsplit: 208 | runs-on: ubuntu-latest 209 | steps: 210 | - name: checkout 211 | run: git clone https://github.com/org/project /home/runner/work/org/project && cd /home/runner/work/org/project 212 | - name: Split repositories 213 | uses: docker://jderusse/gitsplit:latest 214 | with: 215 | args: gitsplit 216 | env: 217 | GH_TOKEN: ${{ secrets.PRIVATE_TOKEN }} 218 | ``` 219 | -------------------------------------------------------------------------------- /gitsplit/cache.go: -------------------------------------------------------------------------------- 1 | package gitsplit 2 | 3 | import ( 4 | "fmt" 5 | "github.com/jderusse/gitsplit/utils" 6 | "github.com/libgit2/git2go" 7 | "github.com/pkg/errors" 8 | log "github.com/sirupsen/logrus" 9 | "path/filepath" 10 | "strings" 11 | ) 12 | 13 | type CachePoolInterface interface { 14 | SaveItem(item *CacheItem) error 15 | GetItem(referenceName string, split Split) (*CacheItem, error) 16 | Load() error 17 | Dump() error 18 | Push() 19 | } 20 | 21 | type NullCachePool struct { 22 | } 23 | 24 | func (c *NullCachePool) SaveItem(item *CacheItem) error { 25 | return nil 26 | } 27 | 28 | func (c *NullCachePool) GetItem(referenceName string, split Split) (*CacheItem, error) { 29 | return &CacheItem{ 30 | flagName: getFlagName(referenceName, split), 31 | }, nil 32 | } 33 | 34 | func (c *NullCachePool) Load() error { 35 | return nil 36 | } 37 | 38 | func (c *NullCachePool) Dump() error { 39 | return nil 40 | } 41 | 42 | func (c *NullCachePool) Push() { 43 | } 44 | 45 | type CachePool struct { 46 | workingSpacePath string 47 | remote *GitRemote 48 | } 49 | 50 | type CacheItem struct { 51 | flagName string 52 | sourceId *git.Oid 53 | targetId *git.Oid 54 | } 55 | 56 | func NewCachePool(workingSpacePath string, remote *GitRemote) *CachePool { 57 | return &CachePool{ 58 | workingSpacePath, 59 | remote, 60 | } 61 | } 62 | 63 | func getFlagName(referenceName string, split Split) string { 64 | return fmt.Sprintf("%s-%s", utils.Hash(referenceName), utils.Hash(strings.Join(split.Prefixes, "-"))) 65 | } 66 | 67 | func (c *CachePool) Load() error { 68 | if err := c.remote.FetchFile("splitsh", "splitsh.db", filepath.Join(c.workingSpacePath, "splitsh.db")); err != nil { 69 | return errors.Wrap(err, "failed to fetch cache") 70 | } 71 | log.Info("Cache loaded") 72 | 73 | return nil 74 | } 75 | 76 | func (c *CachePool) Dump() error { 77 | if !utils.FileExists(filepath.Join(c.workingSpacePath, "splitsh.db")) { 78 | return nil 79 | } 80 | 81 | if err := c.remote.PushFile("splitsh.db", filepath.Join(c.workingSpacePath, "splitsh.db"), "Update splitsh cache", "splitsh"); err != nil { 82 | return errors.Wrap(err, "failed to save cache") 83 | } 84 | log.Info("Cache dumped") 85 | 86 | return nil 87 | } 88 | 89 | func (c *CachePool) Push() { 90 | c.remote.PushAll() 91 | } 92 | 93 | func (c *CachePool) SaveItem(item *CacheItem) error { 94 | if item.SourceId() != nil { 95 | if err := c.remote.AddReference("source-"+item.flagName, item.SourceId()); err != nil { 96 | return errors.Wrapf(err, "failed to create source reference %s for %s", item.flagName, item.SourceId()) 97 | } 98 | } 99 | if item.TargetId() != nil { 100 | if err := c.remote.AddReference("target-"+item.flagName, item.TargetId()); err != nil { 101 | return errors.Wrapf(err, "failed to create target reference %s for %s", item.flagName, item.TargetId()) 102 | } 103 | } 104 | 105 | return nil 106 | } 107 | 108 | func (c *CachePool) GetItem(referenceName string, split Split) (*CacheItem, error) { 109 | flagName := getFlagName(referenceName, split) 110 | sourceReference, err := c.remote.GetReference("source-" + flagName) 111 | if err != nil { 112 | return nil, err 113 | } 114 | 115 | if sourceReference == nil { 116 | return &CacheItem{ 117 | flagName: flagName, 118 | }, nil 119 | } 120 | 121 | targetReference, err := c.remote.GetReference("target-" + flagName) 122 | if err != nil { 123 | return nil, err 124 | } 125 | 126 | if targetReference == nil { 127 | return &CacheItem{ 128 | flagName: flagName, 129 | sourceId: sourceReference.Id, 130 | targetId: nil, 131 | }, nil 132 | } 133 | 134 | return &CacheItem{ 135 | flagName: flagName, 136 | sourceId: sourceReference.Id, 137 | targetId: targetReference.Id, 138 | }, nil 139 | } 140 | 141 | func (c *CacheItem) IsFresh(reference Reference) bool { 142 | if c.sourceId == nil { 143 | return false 144 | } 145 | 146 | return c.sourceId.Equal(reference.Id) 147 | } 148 | 149 | func (c *CacheItem) SourceId() *git.Oid { 150 | return c.sourceId 151 | } 152 | func (c *CacheItem) TargetId() *git.Oid { 153 | return c.targetId 154 | } 155 | func (c *CacheItem) Set(sourceId *git.Oid, targetId *git.Oid) { 156 | c.sourceId = sourceId 157 | c.targetId = targetId 158 | } 159 | -------------------------------------------------------------------------------- /gitsplit/config.go: -------------------------------------------------------------------------------- 1 | package gitsplit 2 | 3 | import ( 4 | "fmt" 5 | "github.com/jderusse/gitsplit/utils" 6 | "github.com/pkg/errors" 7 | log "github.com/sirupsen/logrus" 8 | "gopkg.in/yaml.v2" 9 | "io/ioutil" 10 | "strings" 11 | ) 12 | 13 | type StringCollection []string 14 | type PrefixCollection StringCollection 15 | 16 | type Split struct { 17 | Prefixes PrefixCollection `yaml:"prefix"` 18 | Targets StringCollection `yaml:"target"` 19 | } 20 | 21 | type Config struct { 22 | CacheUrl *GitUrl `yaml:"cache_url"` 23 | ProjectUrl *GitUrl `yaml:"project_url"` 24 | Splits []Split `yaml:"splits"` 25 | Origins []string `yaml:"origins"` 26 | } 27 | 28 | func (s *PrefixCollection) UnmarshalYAML(unmarshal func(interface{}) error) error { 29 | var raw StringCollection 30 | if err := unmarshal(&raw); err != nil { 31 | return err 32 | } 33 | 34 | if len(raw) > 1 { 35 | seen := []string{} 36 | for _, prefix := range raw { 37 | parts := strings.Split(prefix, ":") 38 | if len(parts) != 2 { 39 | return fmt.Errorf("Using several prefixes requires to use the syntax `source:target`. Got %s", prefix) 40 | } 41 | if utils.InArray(seen, parts[1]) { 42 | return fmt.Errorf("Cannot have two prefix splits under the same directory. Got twice %s", parts[1]) 43 | } 44 | seen = append(seen, parts[1]) 45 | } 46 | } 47 | 48 | *s = PrefixCollection(raw) 49 | 50 | return nil 51 | } 52 | 53 | func (s *GitUrl) UnmarshalYAML(unmarshal func(interface{}) error) error { 54 | var raw string 55 | if err := unmarshal(&raw); err != nil { 56 | return err 57 | } 58 | 59 | *s = *ParseUrl(raw) 60 | 61 | return nil 62 | } 63 | 64 | func (s *StringCollection) UnmarshalYAML(unmarshal func(interface{}) error) error { 65 | var rawString string 66 | if err := unmarshal(&rawString); err == nil { 67 | *s = []string{rawString} 68 | 69 | return nil 70 | } 71 | 72 | var rawArray []string 73 | if err := unmarshal(&rawArray); err == nil { 74 | *s = rawArray 75 | 76 | return nil 77 | } 78 | 79 | return fmt.Errorf("expects a string or n array of strings") 80 | } 81 | 82 | func (s *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { 83 | var raw struct { 84 | CacheDir *GitUrl `yaml:"cache_dir"` 85 | CacheUrl *GitUrl `yaml:"cache_url"` 86 | ProjectDir *GitUrl `yaml:"project_dir"` 87 | ProjectUrl *GitUrl `yaml:"project_url"` 88 | Splits []Split `yaml:"splits"` 89 | Origins []string `yaml:"origins"` 90 | } 91 | 92 | if err := unmarshal(&raw); err != nil { 93 | return err 94 | } 95 | 96 | if raw.CacheDir != nil { 97 | log.Error(`The config parameter "cache_dir" is deprecated. Use "cache_url" instead`) 98 | } 99 | 100 | if raw.ProjectDir != nil { 101 | log.Error(`The config parameter "project_dir" is deprecated. Use "project_url" instead`) 102 | } 103 | 104 | if raw.CacheUrl == nil { 105 | raw.CacheUrl = raw.CacheDir 106 | } 107 | if raw.ProjectUrl == nil { 108 | raw.ProjectUrl = raw.ProjectDir 109 | } 110 | 111 | if raw.ProjectUrl == nil { 112 | raw.ProjectUrl = ParseUrl(".") 113 | } 114 | if len(raw.Origins) == 0 { 115 | raw.Origins = []string{".*"} 116 | } 117 | 118 | *s = Config{ 119 | CacheUrl: raw.CacheUrl, 120 | ProjectUrl: raw.ProjectUrl, 121 | Splits: raw.Splits, 122 | Origins: raw.Origins, 123 | } 124 | 125 | return nil 126 | } 127 | 128 | func NewConfigFromFile(filePath string) (*Config, error) { 129 | config := &Config{} 130 | 131 | yamlFile, err := ioutil.ReadFile(utils.ResolvePath(filePath)) 132 | if err != nil { 133 | return nil, errors.Wrap(err, "failed to read config file") 134 | } 135 | 136 | if err = yaml.Unmarshal(yamlFile, &config); err != nil { 137 | return nil, errors.Wrap(err, "failed to load config file") 138 | } 139 | 140 | return config, nil 141 | } 142 | -------------------------------------------------------------------------------- /gitsplit/reference_splitter_lite.go: -------------------------------------------------------------------------------- 1 | package gitsplit 2 | 3 | import ( 4 | "github.com/libgit2/git2go" 5 | lite "github.com/splitsh/lite/splitter" 6 | "strings" 7 | ) 8 | 9 | func NewReferenceSplitterLite(repository *git.Repository) *ReferenceSplitterLite { 10 | return &ReferenceSplitterLite{ 11 | repository: repository, 12 | } 13 | } 14 | 15 | type ReferenceSplitterLite struct { 16 | repository *git.Repository 17 | } 18 | 19 | func formatLitePrefixes(prefixes []string) []*lite.Prefix { 20 | litePrefixes := []*lite.Prefix{} 21 | for _, prefix := range prefixes { 22 | parts := strings.Split(prefix, ":") 23 | from := parts[0] 24 | to := "" 25 | if len(parts) > 1 { 26 | to = parts[1] 27 | } 28 | litePrefixes = append(litePrefixes, &lite.Prefix{From: from, To: to}) 29 | } 30 | 31 | return litePrefixes 32 | } 33 | 34 | func (r *ReferenceSplitterLite) Split(reference string, prefixes []string) (*git.Oid, error) { 35 | config := &lite.Config{ 36 | Path: r.repository.Path(), 37 | Origin: reference, 38 | Prefixes: formatLitePrefixes(prefixes), 39 | Target: "", 40 | Commit: "", 41 | Debug: false, 42 | Scratch: false, 43 | GitVersion: "latest", 44 | } 45 | 46 | result := &lite.Result{} 47 | if err := lite.Split(config, result); err != nil { 48 | return nil, err 49 | } 50 | 51 | return result.Head(), nil 52 | } 53 | -------------------------------------------------------------------------------- /gitsplit/remote.go: -------------------------------------------------------------------------------- 1 | package gitsplit 2 | 3 | import ( 4 | "fmt" 5 | "github.com/gosimple/slug" 6 | "github.com/jderusse/gitsplit/utils" 7 | "github.com/libgit2/git2go" 8 | "github.com/pkg/errors" 9 | log "github.com/sirupsen/logrus" 10 | "io/ioutil" 11 | "os" 12 | "regexp" 13 | "strings" 14 | "sync" 15 | "time" 16 | ) 17 | 18 | type GitRemoteCollection struct { 19 | repository *git.Repository 20 | items map[string]*GitRemote 21 | mutexRemoteList *sync.Mutex 22 | } 23 | 24 | func NewGitRemoteCollection(repository *git.Repository) *GitRemoteCollection { 25 | return &GitRemoteCollection{ 26 | items: make(map[string]*GitRemote), 27 | repository: repository, 28 | mutexRemoteList: &sync.Mutex{}, 29 | } 30 | } 31 | 32 | func (r *GitRemoteCollection) Add(alias string, url string, refs []string) *GitRemote { 33 | remote := NewGitRemote(r.repository, alias, url, refs) 34 | r.items[alias] = remote 35 | 36 | r.mutexRemoteList.Lock() 37 | defer r.mutexRemoteList.Unlock() 38 | remote.Init() 39 | 40 | return remote 41 | } 42 | 43 | func (r *GitRemoteCollection) Get(alias string) (*GitRemote, error) { 44 | if remote, ok := r.items[alias]; !ok { 45 | return nil, errors.New("The remote does not exists") 46 | } else { 47 | return remote, nil 48 | } 49 | 50 | } 51 | 52 | func (r *GitRemoteCollection) Clean() { 53 | knownRemotes := []string{} 54 | for _, remote := range r.items { 55 | knownRemotes = append(knownRemotes, remote.id) 56 | } 57 | 58 | r.mutexRemoteList.Lock() 59 | defer r.mutexRemoteList.Unlock() 60 | 61 | remotes, err := r.repository.Remotes.List() 62 | if err != nil { 63 | return 64 | } 65 | 66 | for _, remoteId := range remotes { 67 | if !utils.InArray(knownRemotes, remoteId) { 68 | log.WithFields(log.Fields{ 69 | "remote": remoteId, 70 | }).Info("Removing remote") 71 | r.repository.Remotes.Delete(remoteId) 72 | } 73 | } 74 | } 75 | 76 | func (r *GitRemoteCollection) Flush() error { 77 | for _, remote := range r.items { 78 | if err := remote.Flush(); err != nil { 79 | return err 80 | } 81 | } 82 | 83 | return nil 84 | } 85 | 86 | type GitRemote struct { 87 | repository *git.Repository 88 | id string 89 | alias string 90 | refs []string 91 | url string 92 | fetched bool 93 | pool *utils.Pool 94 | cacheReferences []Reference 95 | mutexReferences *sync.Mutex 96 | } 97 | 98 | func NewGitRemote(repository *git.Repository, alias string, url string, refs []string) *GitRemote { 99 | id := slug.Make(alias) 100 | if id != alias { 101 | id = id + "-" + utils.Hash(alias) 102 | } 103 | 104 | return &GitRemote{ 105 | repository: repository, 106 | id: id, 107 | alias: alias, 108 | refs: refs, 109 | url: url, 110 | fetched: false, 111 | pool: utils.NewPool(10), 112 | mutexReferences: &sync.Mutex{}, 113 | } 114 | } 115 | 116 | func (r *GitRemote) Init() error { 117 | remotes, err := r.repository.Remotes.List() 118 | if err != nil { 119 | return err 120 | } 121 | 122 | if !utils.InArray(remotes, r.id) { 123 | if _, err := r.repository.Remotes.Create(r.id, os.ExpandEnv(r.url)); err != nil { 124 | return errors.Wrapf(err, "failed to create remote %s", r.alias) 125 | } 126 | } else { 127 | if err := r.repository.Remotes.SetUrl(r.id, os.ExpandEnv(r.url)); err != nil { 128 | return errors.Wrapf(err, "failed to update remote %s", r.alias) 129 | } 130 | } 131 | 132 | return nil 133 | } 134 | 135 | func (r *GitRemote) GetReference(alias string) (*Reference, error) { 136 | references, err := r.GetReferences() 137 | if err != nil { 138 | return nil, errors.Wrap(err, "failed to get reference") 139 | } 140 | 141 | for _, reference := range references { 142 | if reference.Alias == alias { 143 | return &reference, nil 144 | } 145 | } 146 | 147 | return nil, nil 148 | } 149 | 150 | func (r *GitRemote) AddReference(alias string, id *git.Oid) error { 151 | r.mutexReferences.Lock() 152 | defer r.mutexReferences.Unlock() 153 | 154 | r.cacheReferences = nil 155 | for _, ref := range r.refs { 156 | reference, err := r.repository.References.Create(fmt.Sprintf("refs/remotes/%s/%s/%s", r.id, ref, alias), id, true, "") 157 | if err != nil { 158 | return errors.Wrap(err, "failed to add reference") 159 | } 160 | defer reference.Free() 161 | } 162 | 163 | return nil 164 | } 165 | 166 | func (r *GitRemote) GetReferences() ([]Reference, error) { 167 | r.mutexReferences.Lock() 168 | defer r.mutexReferences.Unlock() 169 | 170 | if r.cacheReferences != nil { 171 | return r.cacheReferences, nil 172 | } 173 | 174 | var call func() ([]Reference, error) 175 | if r.fetched { 176 | call = r.getLocalReferences 177 | } else { 178 | call = r.getRemoteReferences 179 | } 180 | 181 | references, err := call() 182 | if err != nil { 183 | return nil, err 184 | } 185 | r.cacheReferences = references 186 | 187 | return references, nil 188 | } 189 | 190 | func (r *GitRemote) getRemoteReferences() ([]Reference, error) { 191 | result, err := utils.GitExec(r.repository.Path(), "ls-remote", r.id) 192 | if err != nil { 193 | return nil, errors.Wrapf(err, "failed to fetch references of %s", r.alias) 194 | } 195 | 196 | references := []Reference{} 197 | cleanShortNameRegexp := regexp.MustCompile(fmt.Sprintf("^refs/")) 198 | cleanAliasRegexp := regexp.MustCompile(fmt.Sprintf("^refs/(%s)/", strings.Join(r.refs, "|"))) 199 | cleanNameRegexp := regexp.MustCompile(fmt.Sprintf("^refs/")) 200 | filterRegexp := regexp.MustCompile(fmt.Sprintf("^refs/(%s)/", strings.Join(r.refs, "|"))) 201 | 202 | for _, line := range strings.Split(result.Stdout, "\n") { 203 | if len(line) == 0 { 204 | continue 205 | } 206 | columns := strings.Split(line, "\t") 207 | if len(columns) != 2 { 208 | return nil, fmt.Errorf("failed to parse reference %s: 2 columns expected", line) 209 | } 210 | referenceId := columns[0] 211 | referenceName := columns[1] 212 | 213 | if !filterRegexp.MatchString(referenceName) { 214 | continue 215 | } 216 | 217 | oid, err := git.NewOid(referenceId) 218 | if err != nil { 219 | return nil, errors.Wrapf(err, "failed to parse reference %s", line) 220 | } 221 | 222 | references = append(references, Reference{ 223 | Alias: cleanAliasRegexp.ReplaceAllString(referenceName, ""), 224 | ShortName: cleanShortNameRegexp.ReplaceAllString(referenceName, ""), 225 | Name: cleanNameRegexp.ReplaceAllString(referenceName, fmt.Sprintf("refs/remotes/%s/", r.id)), 226 | Id: oid, 227 | }) 228 | } 229 | 230 | return references, nil 231 | } 232 | 233 | func (r *GitRemote) getLocalReferences() ([]Reference, error) { 234 | iterator, err := r.repository.NewReferenceIteratorGlob(fmt.Sprintf("refs/remotes/%s/*", r.id)) 235 | if err != nil { 236 | return nil, errors.Wrap(err, "failed to fetch references") 237 | } 238 | 239 | defer iterator.Free() 240 | references := []Reference{} 241 | 242 | reference, err := iterator.Next() 243 | cleanShortNameRegexp := regexp.MustCompile(fmt.Sprintf("^refs/remotes/%s/", r.id)) 244 | cleanAliasRegexp := regexp.MustCompile(fmt.Sprintf("^refs/remotes/%s/(%s)/", r.id, strings.Join(r.refs, "|"))) 245 | filterRegexp := regexp.MustCompile(fmt.Sprintf("^refs/remotes/%s/(%s)/", r.id, strings.Join(r.refs, "|"))) 246 | for err == nil { 247 | if filterRegexp.MatchString(reference.Name()) { 248 | references = append(references, Reference{ 249 | Alias: cleanAliasRegexp.ReplaceAllString(reference.Name(), ""), 250 | ShortName: cleanShortNameRegexp.ReplaceAllString(reference.Name(), ""), 251 | Name: reference.Name(), 252 | Id: reference.Target(), 253 | }) 254 | } 255 | reference, err = iterator.Next() 256 | } 257 | 258 | r.cacheReferences = references 259 | return references, nil 260 | } 261 | 262 | func (r *GitRemote) Fetch() { 263 | r.pool.Push(func() (interface{}, error) { 264 | log.WithFields(log.Fields{ 265 | "remote": r.alias, 266 | "refs": r.refs, 267 | }).Warn("Fetching from remote") 268 | for _, ref := range r.refs { 269 | if _, err := utils.GitExec(r.repository.Path(), "fetch", "--force", "--prune", r.id, fmt.Sprintf("refs/%s/*:refs/remotes/%s/%s/*", ref, r.id, ref)); err != nil { 270 | return nil, errors.Wrapf(err, "failed to update cache of %s", r.alias) 271 | } 272 | } 273 | 274 | r.fetched = true 275 | 276 | return nil, nil 277 | }) 278 | } 279 | 280 | func (r *GitRemote) PushRef(refs string) { 281 | r.pool.Push(func() (interface{}, error) { 282 | log.WithFields(log.Fields{ 283 | "remote": r.alias, 284 | "refs": refs, 285 | }).Warn("Pushing to remote") 286 | if _, err := utils.GitExec(r.repository.Path(), "push", "--force", r.id, refs); err != nil { 287 | return nil, errors.Wrapf(err, "failed to push reference %s", refs) 288 | } 289 | 290 | return nil, nil 291 | }) 292 | } 293 | 294 | func (r *GitRemote) PushAll() { 295 | for _, ref := range r.refs { 296 | r.PushRef(fmt.Sprintf("refs/remotes/%s/%s/*:refs/%s/*", r.id, ref, ref)) 297 | } 298 | } 299 | 300 | func (r *GitRemote) Push(reference Reference, splitId *git.Oid) error { 301 | references, err := r.GetReferences() 302 | if err != nil { 303 | return errors.Wrapf(err, "failed to get references for remote %s", r.alias) 304 | } 305 | 306 | for _, remoteReference := range references { 307 | if remoteReference.Alias == reference.Alias { 308 | if remoteReference.Id.Equal(splitId) { 309 | log.WithFields(log.Fields{ 310 | "remote": r.alias, 311 | }).Info("Already pushed " + reference.Alias) 312 | return nil 313 | } 314 | log.WithFields(log.Fields{ 315 | "remote": r.alias, 316 | }).Warn("Out of date " + reference.Alias) 317 | break 318 | } 319 | } 320 | 321 | r.PushRef(splitId.String() + ":refs/" + reference.ShortName) 322 | 323 | return nil 324 | } 325 | 326 | func (r *GitRemote) FetchFile(referenceName string, fileName string, filePath string) error { 327 | reference, err := r.GetReference("splitsh") 328 | if err != nil { 329 | return errors.Wrapf(err, "failed to fetch file reference %s", referenceName) 330 | } 331 | if reference == nil { 332 | return nil 333 | } 334 | commit, err := r.repository.LookupCommit(reference.Id) 335 | if err != nil { 336 | return errors.Wrapf(err, "failed to find commit %s", reference.Id) 337 | } 338 | defer commit.Free() 339 | tree, err := commit.Tree() 340 | if err != nil { 341 | return errors.Wrapf(err, "failed to fetch commit tree") 342 | } 343 | defer tree.Free() 344 | entry, err := tree.EntryByPath(fileName) 345 | if err != nil { 346 | return errors.Wrapf(err, "failed to fetch file from tree") 347 | } 348 | 349 | odb, err := r.repository.Odb() 350 | if err != nil { 351 | return errors.Wrap(err, "failed to open odb") 352 | } 353 | defer odb.Free() 354 | object, err := odb.Read(entry.Id) 355 | if err != nil { 356 | return errors.Wrap(err, "failed to read from odb") 357 | } 358 | defer object.Free() 359 | if err := ioutil.WriteFile(filePath, object.Data(), os.FileMode(entry.Filemode)); err != nil { 360 | return errors.Wrapf(err, "failed to write file %s", filePath) 361 | } 362 | 363 | return nil 364 | } 365 | 366 | func (r *GitRemote) PushFile(fileName string, filePath string, message string, referenceName string) error { 367 | treeBuilder, err := r.repository.TreeBuilder() 368 | if err != nil { 369 | return errors.Wrap(err, "failed to create treeBuilder") 370 | } 371 | defer treeBuilder.Free() 372 | 373 | file, err := os.Open(filePath) 374 | if err != nil { 375 | return errors.Wrap(err, "failed to open file") 376 | } 377 | defer file.Close() 378 | content, err := ioutil.ReadAll(file) 379 | if err != nil { 380 | return errors.Wrap(err, "failed to read file") 381 | } 382 | odb, err := r.repository.Odb() 383 | if err != nil { 384 | return errors.Wrap(err, "failed to open odb") 385 | } 386 | blobId, err := odb.Write(content, git.ObjectBlob) 387 | if err != nil { 388 | return errors.Wrap(err, "failed to write in odb") 389 | } 390 | if err = treeBuilder.Insert(fileName, blobId, git.FilemodeBlob); err != nil { 391 | return errors.Wrap(err, "failed to insert tree") 392 | } 393 | 394 | treeID, err := treeBuilder.Write() 395 | if err != nil { 396 | return errors.Wrap(err, "failed to write tree") 397 | } 398 | 399 | tree, err := r.repository.LookupTree(treeID) 400 | if err != nil { 401 | return errors.Wrap(err, "failed to find tree") 402 | } 403 | defer tree.Free() 404 | 405 | reference, err := r.GetReference(referenceName) 406 | if err == nil && reference != nil { 407 | return r.replaceFile(reference, message, tree) 408 | } 409 | 410 | return r.insertFile(referenceName, message, tree) 411 | } 412 | 413 | func (r *GitRemote) GetSignature() *git.Signature { 414 | return &git.Signature{ 415 | Name: "gitsplit", 416 | Email: "jeremy+gitsplit@derusse.com", 417 | When: time.Now(), 418 | } 419 | } 420 | 421 | func (r *GitRemote) replaceFile(reference *Reference, message string, tree *git.Tree) error { 422 | r.mutexReferences.Lock() 423 | defer r.mutexReferences.Unlock() 424 | 425 | r.cacheReferences = nil 426 | 427 | commit, err := r.repository.LookupCommit(reference.Id) 428 | if err != nil { 429 | return errors.Wrapf(err, "failed to find commit %s", reference.Id) 430 | } 431 | 432 | sig := r.GetSignature() 433 | if _, err := commit.Amend(reference.Name, sig, sig, message, tree); err != nil { 434 | return err 435 | } 436 | 437 | return nil 438 | } 439 | 440 | func (r *GitRemote) insertFile(referenceName string, message string, tree *git.Tree) error { 441 | r.mutexReferences.Lock() 442 | defer r.mutexReferences.Unlock() 443 | 444 | r.cacheReferences = nil 445 | 446 | sig := r.GetSignature() 447 | if _, err := r.repository.CreateCommit(fmt.Sprintf("refs/remotes/%s/%s/%s", r.id, r.refs[0], referenceName), sig, sig, message, tree); err != nil { 448 | return err 449 | } 450 | 451 | return nil 452 | } 453 | 454 | func (r *GitRemote) Flush() error { 455 | results := r.pool.Wait() 456 | if err := results.FirstError(); err != nil { 457 | return err 458 | } 459 | 460 | return nil 461 | } 462 | -------------------------------------------------------------------------------- /gitsplit/splitter.go: -------------------------------------------------------------------------------- 1 | package gitsplit 2 | 3 | import ( 4 | "github.com/jderusse/gitsplit/utils" 5 | "github.com/libgit2/git2go" 6 | "github.com/pkg/errors" 7 | log "github.com/sirupsen/logrus" 8 | "regexp" 9 | "strings" 10 | ) 11 | 12 | type Reference struct { 13 | Alias string 14 | ShortName string 15 | Name string 16 | Id *git.Oid 17 | } 18 | 19 | type Splitter struct { 20 | config Config 21 | referenceSplitter *ReferenceSplitterLite 22 | workingSpace *WorkingSpace 23 | cachePool CachePoolInterface 24 | } 25 | 26 | func NewSplitter(config Config, workingSpace *WorkingSpace, cachePool CachePoolInterface) *Splitter { 27 | return &Splitter{ 28 | config: config, 29 | workingSpace: workingSpace, 30 | referenceSplitter: NewReferenceSplitterLite(workingSpace.Repository()), 31 | cachePool: cachePool, 32 | } 33 | } 34 | 35 | func (s *Splitter) Split(whitelistReferences []string) error { 36 | remote, err := s.workingSpace.Remotes().Get("origin") 37 | if err != nil { 38 | return err 39 | } 40 | 41 | references, err := remote.GetReferences() 42 | if err != nil { 43 | return errors.Wrap(err, "failed to split references") 44 | } 45 | for _, reference := range references { 46 | for _, referencePattern := range s.config.Origins { 47 | referenceRegexp := regexp.MustCompile(referencePattern) 48 | if !referenceRegexp.MatchString(reference.Alias) { 49 | continue 50 | } 51 | if len(whitelistReferences) > 0 && !utils.InArray(whitelistReferences, reference.Alias) { 52 | continue 53 | } 54 | 55 | for _, split := range s.config.Splits { 56 | if err := s.splitReference(reference, split); err != nil { 57 | return errors.Wrap(err, "failed to split references") 58 | } 59 | } 60 | } 61 | } 62 | 63 | if err := s.workingSpace.Remotes().Flush(); err != nil { 64 | return errors.Wrap(err, "failed to flush references") 65 | } 66 | return nil 67 | } 68 | 69 | func (s *Splitter) splitReference(reference Reference, split Split) error { 70 | flagTemp := "refs/split-temp/" + utils.Hash(reference.Name) + "-" + utils.Hash(strings.Join(split.Prefixes, "-")) 71 | 72 | previousReference, err := s.cachePool.GetItem(reference.Name, split) 73 | if err != nil { 74 | return errors.Wrap(err, "failed to fetch previous state") 75 | } 76 | 77 | contextualLog := log.WithFields(log.Fields{ 78 | "reference": reference.Alias, 79 | "splits": split.Prefixes, 80 | }) 81 | 82 | if previousReference.IsFresh(reference) { 83 | contextualLog.Info("Already splitted") 84 | } else { 85 | contextualLog.Warn("Splitting") 86 | tempReference, err := s.workingSpace.Repository().References.Create(flagTemp, reference.Id, true, "Temporary reference") 87 | if err != nil { 88 | return errors.Wrapf(err, "failed to create temporary reference %s", flagTemp) 89 | } 90 | defer tempReference.Free() 91 | 92 | splitId, err := s.referenceSplitter.Split(flagTemp, split.Prefixes) 93 | if err != nil { 94 | return errors.Wrap(err, "failed to split reference") 95 | } 96 | 97 | err = tempReference.Delete() 98 | if err != nil { 99 | return errors.Wrapf(err, "failed to delete temporary reference %s", flagTemp) 100 | } 101 | 102 | previousReference.Set(reference.Id, splitId) 103 | if err := s.cachePool.SaveItem(previousReference); err != nil { 104 | return errors.Wrapf(err, "failed to cache reference %s", flagTemp) 105 | } 106 | } 107 | 108 | // Reference does not exists 109 | if previousReference.TargetId() == nil { 110 | return nil 111 | } 112 | 113 | for _, target := range split.Targets { 114 | remote, err := s.workingSpace.Remotes().Get(target) 115 | if err != nil { 116 | return err 117 | } 118 | if err := remote.Push(reference, previousReference.TargetId()); err != nil { 119 | return err 120 | } 121 | } 122 | 123 | return nil 124 | } 125 | 126 | func (s *Splitter) getLocalReference(referenceName string) (*git.Oid, error) { 127 | reference, err := s.workingSpace.Repository().References.Dwim(referenceName) 128 | if err != nil { 129 | return nil, nil 130 | } 131 | 132 | return reference.Target(), nil 133 | } 134 | -------------------------------------------------------------------------------- /gitsplit/uri.go: -------------------------------------------------------------------------------- 1 | package gitsplit 2 | 3 | import ( 4 | "github.com/jderusse/gitsplit/utils" 5 | "os" 6 | "strings" 7 | ) 8 | 9 | type GitUrl struct { 10 | scheme string 11 | url string 12 | } 13 | 14 | func (u *GitUrl) IsLocal() bool { 15 | return u.scheme == "file" 16 | } 17 | 18 | func (u *GitUrl) Url() string { 19 | if u.scheme == "" { 20 | return u.SchemelessUrl() 21 | } 22 | 23 | return u.scheme + "://" + u.SchemelessUrl() 24 | } 25 | 26 | func (u *GitUrl) SchemelessUrl() string { 27 | if u.IsLocal() { 28 | return utils.ResolvePath(u.url) 29 | } 30 | 31 | return os.ExpandEnv(u.url) 32 | } 33 | 34 | func ParseUrl(url string) *GitUrl { 35 | parts := strings.SplitN(url, "://", 2) 36 | if len(parts) == 2 { 37 | return &GitUrl{ 38 | scheme: parts[0], 39 | url: parts[1], 40 | } 41 | } 42 | 43 | parts = strings.SplitN(url, "/", 2) 44 | if strings.Index(parts[0], ":") > 0 { 45 | return &GitUrl{ 46 | scheme: "", 47 | url: url, 48 | } 49 | } 50 | 51 | return &GitUrl{ 52 | scheme: "file", 53 | url: url, 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /gitsplit/working_space.go: -------------------------------------------------------------------------------- 1 | package gitsplit 2 | 3 | import ( 4 | "github.com/jderusse/gitsplit/utils" 5 | "github.com/libgit2/git2go" 6 | "github.com/pkg/errors" 7 | log "github.com/sirupsen/logrus" 8 | "io/ioutil" 9 | "os" 10 | ) 11 | 12 | type WorkingSpaceFactory struct { 13 | } 14 | 15 | type WorkingSpace struct { 16 | config Config 17 | repository *git.Repository 18 | remotes *GitRemoteCollection 19 | } 20 | 21 | func NewWorkingSpaceFactory() *WorkingSpaceFactory { 22 | return &WorkingSpaceFactory{} 23 | } 24 | 25 | func (w *WorkingSpaceFactory) CreateWorkingSpace(config Config) (*WorkingSpace, error) { 26 | repository, err := w.getRepository(config) 27 | if err != nil { 28 | return nil, errors.Wrap(err, "failed to create working repository") 29 | } 30 | 31 | workingSpace := &WorkingSpace{ 32 | config: config, 33 | repository: repository, 34 | remotes: NewGitRemoteCollection(repository), 35 | } 36 | 37 | if err := workingSpace.Init(); err != nil { 38 | return nil, err 39 | } 40 | 41 | return workingSpace, nil 42 | } 43 | 44 | func (w *WorkingSpaceFactory) getRepository(config Config) (*git.Repository, error) { 45 | repoPath, err := ioutil.TempDir("", "gitsplit_") 46 | if err != nil { 47 | return nil, errors.Wrap(err, "failed to create working directory") 48 | } 49 | if config.CacheUrl != nil && config.CacheUrl.IsLocal() { 50 | repository, err := git.InitRepository(config.CacheUrl.SchemelessUrl(), true) 51 | if err != nil { 52 | return nil, errors.Wrap(err, "failed to initialize cache repository") 53 | } 54 | repository.Free() 55 | 56 | if err := utils.Copy(config.CacheUrl.SchemelessUrl(), repoPath); err != nil { 57 | return nil, errors.Wrap(err, "failed to create working space from cache") 58 | } 59 | 60 | return git.OpenRepository(repoPath) 61 | } 62 | 63 | log.WithFields(log.Fields{ 64 | "path": repoPath, 65 | }).Info("Create new repository") 66 | return git.InitRepository(repoPath, true) 67 | } 68 | 69 | func (w *WorkingSpace) GetCachePool() (CachePoolInterface, error) { 70 | if w.config.CacheUrl == nil { 71 | return &NullCachePool{}, nil 72 | } 73 | 74 | remote, err := w.Remotes().Get("cache") 75 | if err != nil { 76 | return nil, errors.Wrap(err, "failed to create cache pool") 77 | } 78 | 79 | return NewCachePool(w.repository.Path(), remote), nil 80 | } 81 | 82 | func (w *WorkingSpace) Repository() *git.Repository { 83 | return w.repository 84 | } 85 | 86 | func (w *WorkingSpace) Remotes() *GitRemoteCollection { 87 | return w.remotes 88 | } 89 | 90 | func (w *WorkingSpace) Init() error { 91 | if w.config.CacheUrl != nil && !utils.FileExists(w.config.CacheUrl.SchemelessUrl()) { 92 | log.WithFields(log.Fields{ 93 | "path": w.config.CacheUrl.SchemelessUrl(), 94 | }).Info("Initializing repository") 95 | repository, err := git.InitRepository(w.config.CacheUrl.SchemelessUrl(), true) 96 | if err != nil { 97 | return errors.Wrap(err, "failed to initialize working space") 98 | } 99 | repository.Free() 100 | } 101 | if w.config.CacheUrl != nil { 102 | w.remotes.Add("cache", w.config.CacheUrl.Url(), []string{"split"}).Fetch() 103 | } 104 | w.remotes.Add("origin", w.config.ProjectUrl.Url(), []string{"heads", "tags"}).Fetch() 105 | 106 | for _, split := range w.config.Splits { 107 | for _, target := range split.Targets { 108 | w.remotes.Add(target, target, []string{"heads", "tags"}) 109 | } 110 | } 111 | go w.remotes.Clean() 112 | 113 | if err := w.remotes.Flush(); err != nil { 114 | return err 115 | } 116 | 117 | return nil 118 | } 119 | 120 | func (w *WorkingSpace) Close() { 121 | if err := w.remotes.Flush(); err != nil { 122 | log.Fatal(err) 123 | } 124 | os.RemoveAll(w.repository.Path()) 125 | w.repository.Free() 126 | } 127 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/jderusse/gitsplit/gitsplit" 6 | log "github.com/sirupsen/logrus" 7 | "strings" 8 | ) 9 | 10 | type arrayFlags []string 11 | 12 | func (i *arrayFlags) String() string { 13 | return strings.Join(*i, ", ") 14 | } 15 | 16 | func (i *arrayFlags) Set(value string) error { 17 | *i = append(*i, value) 18 | return nil 19 | } 20 | 21 | var whitelistReferences arrayFlags 22 | 23 | func init() { 24 | flag.Var(&whitelistReferences, "ref", "References to split.") 25 | } 26 | 27 | func handleError(err error) { 28 | log.Fatal(err) 29 | } 30 | 31 | func main() { 32 | flag.Parse() 33 | 34 | config, err := gitsplit.NewConfigFromFile(".gitsplit.yml") 35 | if err != nil { 36 | handleError(err) 37 | } 38 | 39 | workingSpaceFactory := gitsplit.NewWorkingSpaceFactory() 40 | 41 | workingSpace, err := workingSpaceFactory.CreateWorkingSpace(*config) 42 | defer workingSpace.Close() 43 | if err != nil { 44 | handleError(err) 45 | } 46 | 47 | cachePool, err := workingSpace.GetCachePool() 48 | if err != nil { 49 | handleError(err) 50 | } 51 | if err := cachePool.Load(); err != nil { 52 | handleError(err) 53 | } 54 | 55 | splitter := gitsplit.NewSplitter(*config, workingSpace, cachePool) 56 | if err := splitter.Split(whitelistReferences); err != nil { 57 | handleError(err) 58 | } 59 | 60 | if err := cachePool.Dump(); err != nil { 61 | handleError(err) 62 | } 63 | 64 | cachePool.Push() 65 | } 66 | -------------------------------------------------------------------------------- /utils/array.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import () 4 | 5 | func InArray(arr []string, str string) bool { 6 | for _, a := range arr { 7 | if a == str { 8 | return true 9 | } 10 | } 11 | 12 | return false 13 | } 14 | -------------------------------------------------------------------------------- /utils/exec.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | log "github.com/sirupsen/logrus" 7 | "os/exec" 8 | "strings" 9 | "syscall" 10 | ) 11 | 12 | type ExecResut struct { 13 | ExitCode int 14 | Stdout string 15 | Stderr string 16 | Output string 17 | } 18 | 19 | func Exec(name string, arg ...string) ExecResut { 20 | cmd := exec.Command(name, arg...) 21 | result := ExecResut{} 22 | 23 | var stdoutBuffer bytes.Buffer 24 | var stderrBuffer bytes.Buffer 25 | cmd.Stdout = &stdoutBuffer 26 | cmd.Stderr = &stderrBuffer 27 | err := cmd.Run() 28 | 29 | result.Stdout = stdoutBuffer.String() 30 | result.Stderr = stderrBuffer.String() 31 | result.Output = result.Stdout + result.Stderr 32 | if err != nil { 33 | if exitError, ok := err.(*exec.ExitError); ok { 34 | ws := exitError.Sys().(syscall.WaitStatus) 35 | result.ExitCode = ws.ExitStatus() 36 | } else { 37 | result.ExitCode = 128 38 | if result.Output == "" { 39 | result.Output = err.Error() 40 | } 41 | } 42 | } else { 43 | ws := cmd.ProcessState.Sys().(syscall.WaitStatus) 44 | result.ExitCode = ws.ExitStatus() 45 | } 46 | return result 47 | } 48 | 49 | func GitExec(repository string, command string, arg ...string) (ExecResut, error) { 50 | result := Exec("git", append([]string{"--git-dir", repository, command}, arg...)...) 51 | if result.ExitCode != 0 { 52 | return result, fmt.Errorf(result.Output) 53 | } 54 | 55 | log.Debug(strings.Join(append([]string{"git", command}, arg...), " ")) 56 | log.Debug(result.Output) 57 | 58 | return result, nil 59 | } 60 | -------------------------------------------------------------------------------- /utils/file.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "github.com/otiai10/copy" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | ) 9 | 10 | func FileExists(path string) bool { 11 | if _, err := os.Stat(path); os.IsNotExist(err) { 12 | return false 13 | } 14 | 15 | return true 16 | } 17 | 18 | func Copy(source string, target string) error { 19 | return copy.Copy(source, target) 20 | } 21 | 22 | func ResolvePath(path string) string { 23 | path = os.ExpandEnv(path) 24 | if path == "~" || strings.HasPrefix(path, "~/") { 25 | path = strings.Replace(path, "~", os.Getenv("HOME"), 1) 26 | } 27 | 28 | if filepath.IsAbs(path) { 29 | return path 30 | } 31 | 32 | pwd, err := os.Getwd() 33 | if err != nil { 34 | return path 35 | } 36 | 37 | return filepath.Join(pwd, path) 38 | } 39 | -------------------------------------------------------------------------------- /utils/hash.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | ) 7 | 8 | func Hash(input string) string { 9 | sha_256 := sha256.New() 10 | sha_256.Write([]byte(input)) 11 | 12 | return hex.EncodeToString(sha_256.Sum(nil)) 13 | } 14 | -------------------------------------------------------------------------------- /utils/pool.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "gopkg.in/go-playground/pool.v3" 5 | ) 6 | 7 | type Pool struct { 8 | pool pool.Pool 9 | batch pool.Batch 10 | } 11 | 12 | type PoolResult struct { 13 | Value func() interface{} 14 | Error func() error 15 | } 16 | 17 | type PoolResults []PoolResult 18 | 19 | func NewPool(maxSize uint) *Pool { 20 | p := &Pool{ 21 | pool: pool.NewLimited(maxSize), 22 | } 23 | 24 | p.start() 25 | 26 | return p 27 | } 28 | 29 | func (p *Pool) start() { 30 | p.batch = p.pool.Batch() 31 | } 32 | 33 | func (p *PoolResults) FirstError() error { 34 | for _, result := range *p { 35 | if err := result.Error(); err != nil { 36 | return err 37 | } 38 | } 39 | 40 | return nil 41 | } 42 | 43 | func (p *Pool) Wait() PoolResults { 44 | p.batch.QueueComplete() 45 | 46 | results := []PoolResult{} 47 | 48 | for result := range p.batch.Results() { 49 | results = append(results, PoolResult{ 50 | Value: result.Value, 51 | Error: result.Error, 52 | }) 53 | } 54 | 55 | p.start() 56 | 57 | return results 58 | } 59 | 60 | func (p *Pool) Close() { 61 | p.pool.Close() 62 | } 63 | 64 | func (p *Pool) Push(callback func() (interface{}, error)) { 65 | p.batch.Queue(func(wu pool.WorkUnit) (interface{}, error) { 66 | if wu.IsCancelled() { 67 | return nil, nil 68 | } 69 | 70 | return callback() 71 | }) 72 | } 73 | --------------------------------------------------------------------------------