├── .gitignore ├── Makefile ├── Dockerfile ├── Dockerfile.multi-stage ├── util.go ├── LICENSE.md ├── file_sorter.go ├── main.go ├── README.md └── command.go /.gitignore: -------------------------------------------------------------------------------- 1 | pkg/* 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all test clean build install 2 | 3 | GOFLAGS ?= $(GOFLAGS:) 4 | 5 | all: install test 6 | 7 | 8 | build: 9 | @gox -os="linux darwin" -arch="amd64" \ 10 | -output "pkg/{{.OS}}_{{.Arch}}/hacher" \ 11 | . 12 | 13 | install: 14 | @go get github.com/mitchellh/gox 15 | @go get $(GOFLAGS) 16 | 17 | test: install 18 | @go test $(GOFLAGS) 19 | 20 | clean: 21 | @go clean $(GOFLAGS) -i 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:latest 2 | 3 | # If RELEASE is not passed as a build arg, go with master 4 | ARG RELEASE=master 5 | WORKDIR /go 6 | ENV GOBIN $GOPATH/bin 7 | RUN echo "Using Hacher ${RELEASE}" \ 8 | && wget -q https://github.com/Dockbit/hacher/archive/${RELEASE}.tar.gz -O hacher-${RELEASE}.tar.gz \ 9 | && tar xf hacher-${RELEASE}.tar.gz \ 10 | && cd hacher-* \ 11 | && make install && make build \ 12 | && cp /go/bin/hacher-* /hacher 13 | ENTRYPOINT ["/hacher"] 14 | -------------------------------------------------------------------------------- /Dockerfile.multi-stage: -------------------------------------------------------------------------------- 1 | FROM golang:latest as builder 2 | 3 | # If RELEASE is not passed as a build arg, go with master 4 | ARG RELEASE=master 5 | WORKDIR /go 6 | ENV GOBIN $GOPATH/bin 7 | RUN echo "Using Hacher ${RELEASE}" \ 8 | && wget -q https://github.com/Dockbit/hacher/archive/${RELEASE}.tar.gz -O hacher-${RELEASE}.tar.gz \ 9 | && tar xf hacher-${RELEASE}.tar.gz \ 10 | && cd hacher-* \ 11 | && make install && make build 12 | 13 | FROM alpine:latest 14 | 15 | COPY --from=0 /go/bin/hacher-* /hacher 16 | ENTRYPOINT ["/hacher"] 17 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/mgutz/ansi" 8 | ) 9 | 10 | func printInfo(message string, args ...interface{}) { 11 | if verbose { 12 | logger.Println(colorizeMessage("green", "info:", message, args...)) 13 | } 14 | } 15 | 16 | func printFatal(message string, args ...interface{}) { 17 | logger.Println(colorizeMessage("red", "error:", message, args...)) 18 | os.Exit(1) 19 | } 20 | 21 | func checkError(err error) { 22 | if err != nil { 23 | printFatal(err.Error()) 24 | } 25 | } 26 | 27 | func colorizeMessage(color, prefix, message string, args ...interface{}) string { 28 | prefResult := "" 29 | if prefix != "" { 30 | prefResult = ansi.Color(prefix, color+"+b") + " " + ansi.ColorCode("reset") 31 | } 32 | return prefResult + ansi.Color(fmt.Sprintf(message, args...), color) + ansi.ColorCode("reset") 33 | } 34 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Dockbit Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /file_sorter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "regexp" 8 | "sort" 9 | ) 10 | 11 | type File struct { 12 | Path string 13 | os.FileInfo 14 | } 15 | 16 | type Files []*File 17 | 18 | type ByName struct{ Files } 19 | type ByMtime struct{ Files } 20 | 21 | func (a Files) Len() int { return len(a) } 22 | func (a Files) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 23 | 24 | func (a ByName) Less(i, j int) bool { return a.Files[i].Path < a.Files[j].Path } 25 | 26 | func (a ByMtime) Less(i, j int) bool { 27 | return a.Files[i].ModTime().Before(a.Files[j].ModTime()) 28 | } 29 | 30 | func (f File) String() string { 31 | return fmt.Sprintf("{%s: %v %d}", f.Path, f.ModTime(), f.Size()) 32 | } 33 | 34 | /* 35 | * Sorts files by descending mtime with an optional regex filtering. 36 | */ 37 | func fileSorter(path string, filter ...string) Files { 38 | var files Files 39 | 40 | err := filepath.Walk(path, 41 | func(path string, info os.FileInfo, err error) error { 42 | if err == nil && info.Mode().IsRegular() { 43 | matched := true 44 | 45 | // filter by regex 46 | if len(filter) > 0 { 47 | matched, _ = regexp.MatchString(filter[0], filepath.Base(path)) 48 | } 49 | if matched { 50 | files = append(files, &File{Path: path, FileInfo: info}) 51 | } 52 | } 53 | return nil 54 | }, 55 | ) 56 | checkError(err) 57 | sort.Sort(sort.Reverse(ByMtime{files})) 58 | 59 | return files 60 | } 61 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "strconv" 7 | 8 | "github.com/codegangsta/cli" 9 | ) 10 | 11 | const ( 12 | dirMode os.FileMode = 0755 // Default mode for directories 13 | ) 14 | 15 | var ( 16 | logger = log.New(os.Stderr, "", 0) 17 | verbose = false 18 | 19 | CachePath = os.Getenv("HACHER_PATH") 20 | CacheKeep = 3 21 | ) 22 | 23 | func initClient() { 24 | if len(CachePath) < 1 { 25 | printFatal("Env variable HACHER_PATH is not set. Point it to the cache directory.") 26 | } 27 | if len(os.Getenv("HACHER_KEEP")) > 0 { 28 | if i, err := strconv.Atoi(os.Getenv("HACHER_KEEP")); err == nil { 29 | CacheKeep = i 30 | } 31 | } 32 | } 33 | 34 | func main() { 35 | 36 | initClient() 37 | 38 | app := cli.NewApp() 39 | 40 | app.Name = "Hacher" 41 | app.Usage = "A simple CLI tool to cache project artifacts." 42 | app.Author = "The Dockbit Team" 43 | app.Email = "team@dockbit.com" 44 | app.Version = "0.1.0" 45 | 46 | // Alphabetically ordered list of commands 47 | app.Flags = []cli.Flag{ 48 | cli.BoolFlag{ 49 | Name: "verbose, x", 50 | Usage: "Verbose mode", 51 | }, 52 | } 53 | 54 | app.Commands = []cli.Command{ 55 | 56 | { 57 | Name: "get", 58 | Usage: "Gets cache content by a given key.", 59 | Action: cmdGet, 60 | Flags: []cli.Flag{ 61 | 62 | cli.StringFlag{ 63 | Name: "k, key", 64 | Usage: "Cache key", 65 | }, 66 | 67 | cli.StringFlag{ 68 | Name: "f, file", 69 | Usage: "Path to comma-separated dependency file(s) to track for changes.", 70 | }, 71 | 72 | cli.StringFlag{ 73 | Name: "e, env", 74 | Usage: "Comma-separated dependency env variable(s) to track for changes.", 75 | }, 76 | }, 77 | }, 78 | { 79 | Name: "set", 80 | Usage: "Saves cache content for a given key.", 81 | Action: cmdSet, 82 | Flags: []cli.Flag{ 83 | 84 | cli.StringFlag{ 85 | Name: "k, key", 86 | Usage: "Cache key", 87 | }, 88 | 89 | cli.StringFlag{ 90 | Name: "f, file", 91 | Usage: "Path to comma-separated dependency file(s) to track for changes.", 92 | }, 93 | 94 | cli.StringFlag{ 95 | Name: "e, env", 96 | Usage: "Comma-separated dependency env variable(s) to track for changes.", 97 | }, 98 | }, 99 | }, 100 | } 101 | 102 | app.Run(os.Args) 103 | } 104 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Hacher :hocho: 2 | 3 | ```hacher``` is a simple CLI tool to cache project artifacts. It tracks the changes to one or more dependency files, and if any of them has changed, caches related artifacts directory. For example, you could cache ```bundle``` directory on [Gemfile](http://bundler.io/man/gemfile.5.html) changes or a ```node_modules``` when [package.json](https://docs.npmjs.com/files/package.json) changes. 4 | 5 | ## Installation 6 | 7 | Install [Golang](https://golang.org/doc/install) and: 8 | 9 | make install 10 | 11 | ```hacher``` needs several environment variables to operate: 12 | 13 | * ```HACHER_PATH``` - The path where cached artifacts will be stored. 14 | * ```HACHER_KEEP``` - The number of caches to keep, defaults to ```3```. You probably won't need more, since it's only useful if someone reverts the dependency file, hence cache could be reused. 15 | 16 | ### Building in Docker 17 | 18 | You can use the [Dockerfile](./Dockerfile) to download and install the hacher binary inside a Docker image. This is handy in case you don't have the Go environment setup locally. 19 | 20 | Specifying the [version](https://github.com/Dockbit/hacher/releases) to install is available via the `RELEASE` Docker build argument. 21 | 22 | ```docker build --tag hacher --build-arg RELEASE=v0.1.0 .``` 23 | 24 | ## Usage 25 | 26 | Let's say you wanted to speed up ```npm install``` during your CI builds, so you could: 27 | 28 | # Get the cached version of node_modules to the current directory 29 | hacher get -k node_modules -f package.json 30 | 31 | npm install 32 | 33 | # Cache node_modules on package.json changes 34 | hacher set -k node_modules -f package.json ./node_modules 35 | 36 | To get more help: 37 | 38 | hacher --help 39 | 40 | ### Running in Docker 41 | 42 | If you previously baked hacher in a Docker image, you can run it by exposing the files to be cached as Docker volumes to the container. 43 | 44 | ``` 45 | docker run --volume $PWD:/source \ 46 | --volume $PWD/hacher_cache:/cache \ 47 | --env HACHER_PATH=/cache \ 48 | --workdir /source \ 49 | hacher set -k node_modules -f package.json ./node_modules 50 | ``` 51 | 52 | ## ToDo 53 | 54 | Here are some things to be added later. Contributions are welcome! 55 | 56 | * Optionally store cache in Amazon S3 57 | * Use Golang native archive utilities 58 | * Conditional exec ```hacher exec ...``` to run operation only if there is no cache 59 | * Some testing would be nice :innocent: 60 | 61 | 62 | ## License 63 | 64 | See the [LICENSE](LICENSE.md) file for license rights and limitations (MIT). 65 | 66 | -------------------------------------------------------------------------------- /command.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "github.com/codegangsta/cli" 7 | "io/ioutil" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | "strings" 12 | "bytes" 13 | "sort" 14 | ) 15 | 16 | func cmdGet(c *cli.Context) { 17 | 18 | verbose = c.GlobalBool("verbose") 19 | 20 | path := c.Args().Get(0) 21 | key := c.String("key") 22 | files := strings.Split(c.String("file"), ",") 23 | envs := strings.Split(c.String("env"), ",") 24 | 25 | if len(path) < 1 { 26 | path = "." // default to current directory 27 | } 28 | 29 | if len(key) < 1 { 30 | key = strings.ToLower(filepath.Base(files[0])) 31 | } 32 | hash := checksum(files, envs) 33 | fullPath := filepath.Join(CachePath, strings.Join([]string{key, hash}, "-")) + ".tar.gz" 34 | 35 | // get cache if exists 36 | if _, err := os.Stat(fullPath); err == nil { 37 | printInfo("Fetching cache '%s'. Please, wait...", key) 38 | 39 | args := []string{ 40 | "-xzf", 41 | fullPath, 42 | "-C", 43 | path, 44 | } 45 | 46 | err := exec.Command("tar", args...).Run() 47 | checkError(err) 48 | } 49 | } 50 | 51 | func cmdSet(c *cli.Context) { 52 | 53 | verbose = c.GlobalBool("verbose") 54 | 55 | path := c.Args().Get(0) 56 | key := c.String("key") 57 | files := strings.Split(c.String("file"), ",") 58 | envs := strings.Split(c.String("env"), ",") 59 | 60 | if len(path) < 1 { 61 | printFatal("Path to content is not provided as an argument.") 62 | } 63 | 64 | if _, err := os.Stat(path); os.IsNotExist(err) { 65 | printFatal("Content '%s' does not exist.", path) 66 | } 67 | 68 | if _, err := os.Stat(CachePath); os.IsNotExist(err) { 69 | if os.MkdirAll(CachePath, dirMode) != nil { 70 | printFatal("Couldn't create cache directory. "+ 71 | "Is the %s directory writable?", CachePath) 72 | } 73 | } 74 | 75 | if len(key) < 1 { 76 | key = strings.ToLower(filepath.Base(files[0])) 77 | } 78 | 79 | hash := checksum(files, envs) 80 | fullPath := filepath.Join(CachePath, strings.Join([]string{key, hash}, "-")) + ".tar.gz" 81 | 82 | // cache contents only if it doesn't exist already 83 | if _, err := os.Stat(fullPath); os.IsNotExist(err) { 84 | printInfo("Caching '%s'. Please, wait...", key) 85 | 86 | args := []string{ 87 | "-czf", 88 | fullPath, 89 | path, 90 | } 91 | 92 | err := exec.Command("tar", args...).Run() 93 | checkError(err) 94 | } 95 | clean(key) 96 | } 97 | 98 | /* 99 | * Calculates SHA256 checksum of an array of files and/or env vars 100 | * 101 | * Returns The String checksum of all files 102 | */ 103 | func checksum(files []string, envs []string) string { 104 | if len(files[0]) < 1 { 105 | printFatal("At least one dependency file is required.") 106 | } 107 | 108 | var buffer bytes.Buffer 109 | 110 | // first go thru files 111 | sort.Strings(files) 112 | for _, file := range files { 113 | if _, err := os.Stat(file); os.IsNotExist(err) { 114 | printFatal("Dependency file '%s' does not exist.", file) 115 | } 116 | contents, err := ioutil.ReadFile(file) 117 | checkError(err) 118 | buffer.Write(contents) 119 | } 120 | 121 | // then check any environment variables 122 | sort.Strings(envs) 123 | for _, v := range envs { 124 | envVar := os.Getenv(v) 125 | if len(envVar) > 0 { 126 | buffer.WriteString(envVar) 127 | } 128 | } 129 | 130 | hasher := sha256.New() 131 | hasher.Write(buffer.Bytes()) 132 | return hex.EncodeToString(hasher.Sum(nil)) 133 | } 134 | 135 | /* 136 | * Cleans old caches 137 | */ 138 | func clean(key string) { 139 | files := fileSorter(CachePath, "^"+key+"-") 140 | 141 | for index, file := range files { 142 | if index+1 > CacheKeep { 143 | os.Remove(file.Path) 144 | } 145 | } 146 | } 147 | --------------------------------------------------------------------------------